avoid dupes in linklist, sanitize urls

parser.py

@@ -28,7 +28,7 @@ class LinkList():
                 s.parse_categories()
                 s.generate_articles()
     
-                self.links=[a.url for a in s.articles]
+                self.links=list(set([a.url for a in s.articles])) #avoid dupes
             else:
                 f=FrontPage(url,selector)
                 self.links=f.links
@@ -36,13 +36,13 @@ class LinkList():
         self.links=self.purgeLinks(self.links)
 
     def purgeLinks(self,l):
-        return [ link for link in l if not "presslist" in link.lower() and not "videolist" in link.lower() ]
+        return [ sanitizeUrl(link) for link in l if not "presslist" in link.lower() and not "videolist" in link.lower() ]
 
 class News():
     r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
     r_noise=re.compile(r"posted|Product|Google",flags=re.IGNORECASE)
     def __init__(self,url,lang="en"):
-        url=self.sanitizeUrl(url)
+        url=sanitizeUrl(url)
         a = Article(url, language=lang,keep_article_html=True)
         a.download()
         a.parse()
@@ -59,8 +59,6 @@ class News():
         print(a.movies)
         print(htmlmin.minify(a.article_html,remove_empty_space=True))
 
-    def sanitizeUrl(self,url):
-        return url.replace("?share=google-plus-1","")
 
     def dedup(self,val):
         return list(set(val))
@@ -111,15 +109,22 @@ l=None
 #l=LinkList("https://news.starbucks.com/feeds/news")
 #l=LinkList("http://www.techinsider.io/rss",rss=True)
 
-l=LinkList("https://newsroom.uber.com/feed/", rss=True)
+#l=LinkList("https://newsroom.uber.com/feed/", rss=True)
 
-if l is not None:
-    for a in l.links:
-        print(a)
-    print(len(l.links))
-    n=News(l.links[0])
+#if l is not None:
+#    for a in l.links:
+#        print(a)
+#    print(len(l.links))
+#    n=News(l.links[0])
+#
+#sys.exit(1)
+
+unsharer=re.compile(r"\?share=.+$",flags=re.IGNORECASE)
+def sanitizeUrl(url):
+    global unsharer
+    return re.sub(unsharer, "",url)
+    #return url.replace("?share=google-plus-1","").replace("?share=facebook","")
 
-sys.exit(1)
 def sanitizeSelector(s):
     s=s.replace(">a","> a")
     if re.match(r'=\w+\]',s) is None:
@@ -130,14 +135,20 @@ def sanitizeSelector(s):
     print(ret)
     return ret
 
+d=db()
 for s in d.sources():
-    s["title"]=sanitizeSelector(s["title"])
-
-    l=LinkList(s["link"],s["title"])
-    if len(l.links)==0:
-        print(s["link"])
-        print(s["title"])
-        print("###################################")
-
-
-#n=News("http://www.bloomberg.com/news/articles/2016-03-04/the-mystery-madoff-victims-who-left-2-5-billion-on-the-table")
+    print(s["name"])
+    sel=None
+    rss=False
+    if "selector" in s:
+        sel=s["selector"]
+    if "rss" in s:
+        rss=bool(["rss"])
+    l=LinkList(s["link"],sel,rss=rss)
+
+    if l is not None:
+        for a in l.links:
+            print(a)
+        print(len(l.links))
+        n=News(l.links[0])
+    break