|
|
@@ -101,10 +101,11 @@ class News():
|
|
|
return a.replace("_", " ").lower()
|
|
|
|
|
|
|
|
|
-unsharer=re.compile(r"\?share=.+$",flags=re.IGNORECASE)
|
|
|
+unsharer=re.compile(r"(\?share=|#).+$",flags=re.IGNORECASE)
|
|
|
def sanitizeUrl(url):
|
|
|
global unsharer
|
|
|
- return re.sub(unsharer, "",url)
|
|
|
+ ret = re.sub(unsharer, "",url)
|
|
|
+ return ret
|
|
|
|
|
|
def sanitizeSelector(s):
|
|
|
s=s.replace(">a","> a")
|
|
|
@@ -127,8 +128,9 @@ for s in d.sources():
|
|
|
|
|
|
if l is not None:
|
|
|
for a in l.links:
|
|
|
- print(a)
|
|
|
+ if not d.article_exists(a):
|
|
|
+ print(a)
|
|
|
+ n=News(a,s["_id"])
|
|
|
+ d.insert_article(n.get())
|
|
|
print(len(l.links))
|
|
|
- n=News(l.links[0],s["_id"])
|
|
|
- print(n.get())
|
|
|
break
|
