refactor

functions.py

@@ -0,0 +1,16 @@
+import re
+unsharer=re.compile(r"(\?share=|#).+$",flags=re.IGNORECASE)
+def sanitizeUrl(url):
+    global unsharer
+    ret = re.sub(unsharer, "",url)
+    return ret
+
+def sanitizeSelector(s):
+    s=s.replace(">a","> a")
+    if re.match(r'=\w+\]',s) is None:
+        return s
+
+    ret=re.sub(r"=(\w+)]", r'="\1"]', s)
+    return ret
+
+

linklist.py

@@ -0,0 +1,36 @@
+import feedparser
+from frontpage import FrontPage
+from functions import sanitizeUrl
+import newspaper
+class LinkList():
+    def __init__(self,url,selector=None,rss=False):
+        self.links=[]
+        if rss:
+            feed = feedparser.parse(url)
+            if feed["bozo"]:
+                print("RSS ERROR. PANIC")
+            #print(feed["bozo"]) FIXME: If bozo==1 => error
+            self.links=[i["link"] for i in feed["items"]]
+            print(self.links)
+        else:
+            if selector is None:
+                #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida
+                s = newspaper.Source(url,memoize_articles=False)
+                s.download()
+                s.parse()
+                s.set_categories()
+                s.download_categories()
+                s.parse_categories()
+                s.generate_articles()
+    
+                self.links=[a.url for a in s.articles] 
+            else:
+                f=FrontPage(url,selector)
+                self.links=f.links
+        self.links=self.purgeLinks(self.links)
+        self.links=list(set(self.links)) #avoid dupes
+
+    def purgeLinks(self,l):
+        return [ sanitizeUrl(link) for link in l if not "presslist" in link.lower() and not "videolist" in link.lower() ]
+
+

news.py

@@ -0,0 +1,86 @@
+import re
+import htmlmin
+import datetime
+
+from functions import sanitizeUrl
+from newspaper import Article
+
+class News():
+    r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
+    r_noise=re.compile(r"posted|Product|Google",flags=re.IGNORECASE)
+    
+    URL=""
+    AUTHORS=[]
+    SCRAP_DATE=None 
+    PUBLISH_DATE=None
+    SUMMARY=""
+    TEXT=""
+    HTML=""
+    ERROR=False
+    SOURCE=""
+    WEIGHT_SUM=0
+    MATCHES=False
+    MATCHING_KW=[]
+
+    def __init__(self,url,source,kw,lang="en"):
+        url=sanitizeUrl(url)
+        self.URL=url
+        self.SOURCE=source
+
+        a = Article(url, language=lang,keep_article_html=True)
+        try:
+            a.download()
+            a.parse()
+            return
+        except Exception:
+            print("ERROR parsing/downloading article")
+            ERROR=True
+            return
+        a.nlp()
+        
+        a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
+        
+        self.SCRAP_DATE=datetime.datetime.now()
+        self.AUTHORS=a.authors
+        self.PUBLISH_DATE=a.publish_date
+        self.SUMMARY=a.summary
+        self.TEXT=a.text
+        self.HTML=htmlmin.minify(a.article_html,remove_empty_space=True)
+        for k in kw:
+            if re.match(k["pattern"], self.TEXT):
+                self.WEIGHT_SUM+=k["weight"]
+                self.MATCHING_KW.append(k["kw"])
+                self.MATCHES=True
+            
+
+    def get(self):
+        return {
+                "url": self.URL,
+                "source": self.SOURCE,
+                "authors": self.AUTHORS,
+                "scrap_date": self.SCRAP_DATE,
+                "publish_date": self.PUBLISH_DATE,
+                "summary": self.SUMMARY,
+                "text": self.TEXT,
+                "matches": self.MATCHES,
+                "weight": self.WEIGHT_SUM,
+                "kw": self.MATCHING_KW,
+                "html": self.HTML 
+                }
+
+    def dedup(self,val):
+        return list(set(val))
+
+    def isNoise(self,v):
+        if self.r_noise.match(v):
+            return True
+        return False
+    def isComment(self,v):
+        if self.r.match(v):
+            return True
+        return False
+    
+    def fix_author(self,a):
+        return a.replace("_", " ").lower()
+
+

parser.py

@@ -1,146 +1,11 @@
 #!/usr/bin/python3 
-import sys
-import re
-import newspaper
-import pymongo
-import htmlmin
-import feedparser
-import datetime
 import threading
-
 from queue import Queue 
-from frontpage import FrontPage
-from newspaper import Article
+
 from db import db
 from source import Source
-
-class LinkList():
-    def __init__(self,url,selector=None,rss=False):
-        self.links=[]
-        if rss:
-            feed = feedparser.parse(url)
-            if feed["bozo"]:
-                print("RSS ERROR. PANIC")
-            #print(feed["bozo"]) FIXME: If bozo==1 => error
-            self.links=[i["link"] for i in feed["items"]]
-            print(self.links)
-        else:
-            if selector is None:
-                #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida
-                s = newspaper.Source(url,memoize_articles=False)
-                s.download()
-                s.parse()
-                s.set_categories()
-                s.download_categories()
-                s.parse_categories()
-                s.generate_articles()
-    
-                self.links=[a.url for a in s.articles] 
-            else:
-                f=FrontPage(url,selector)
-                self.links=f.links
-        self.links=self.purgeLinks(self.links)
-        self.links=list(set(self.links)) #avoid dupes
-
-    def purgeLinks(self,l):
-        return [ sanitizeUrl(link) for link in l if not "presslist" in link.lower() and not "videolist" in link.lower() ]
-
-class News():
-    r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
-    r_noise=re.compile(r"posted|Product|Google",flags=re.IGNORECASE)
-    
-    URL=""
-    AUTHORS=[]
-    SCRAP_DATE=None 
-    PUBLISH_DATE=None
-    SUMMARY=""
-    TEXT=""
-    HTML=""
-    ERROR=False
-    SOURCE=""
-    WEIGHT_SUM=0
-    MATCHES=False
-    MATCHING_KW=[]
-
-    def __init__(self,url,source,kw,lang="en"):
-        url=sanitizeUrl(url)
-        self.URL=url
-        self.SOURCE=source
-
-        a = Article(url, language=lang,keep_article_html=True)
-        try:
-            a.download()
-            a.parse()
-        except newspaper.article.ArticleException:
-            print("ERROR parsing article")
-            ERROR=True
-            return
-        except Exception:
-            print("ERROR downloading article")
-            ERROR=True
-            return
-        a.nlp()
-        
-        a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
-        
-        self.SCRAP_DATE=datetime.datetime.now()
-        self.AUTHORS=a.authors
-        self.PUBLISH_DATE=a.publish_date
-        self.SUMMARY=a.summary
-        self.TEXT=a.text
-        self.HTML=htmlmin.minify(a.article_html,remove_empty_space=True)
-        for k in kw:
-            if re.match(k["pattern"], self.TEXT):
-                self.WEIGHT_SUM+=k["weight"]
-                self.MATCHING_KW.append(k["kw"])
-                self.MATCHES=True
-            
-
-    def get(self):
-        return {
-                "url": self.URL,
-                "source": self.SOURCE,
-                "authors": self.AUTHORS,
-                "scrap_date": self.SCRAP_DATE,
-                "publish_date": self.PUBLISH_DATE,
-                "summary": self.SUMMARY,
-                "text": self.TEXT,
-                "matches": self.MATCHES,
-                "weight": self.WEIGHT_SUM,
-                "kw": self.MATCHING_KW,
-                "html": self.HTML 
-                }
-
-    def dedup(self,val):
-        return list(set(val))
-
-    def isNoise(self,v):
-        if self.r_noise.match(v):
-            return True
-        return False
-    def isComment(self,v):
-        if self.r.match(v):
-            return True
-        return False
-    
-    def fix_author(self,a):
-        return a.replace("_", " ").lower()
-
-
-unsharer=re.compile(r"(\?share=|#).+$",flags=re.IGNORECASE)
-def sanitizeUrl(url):
-    global unsharer
-    ret = re.sub(unsharer, "",url)
-    return ret
-
-def sanitizeSelector(s):
-    s=s.replace(">a","> a")
-    if re.match(r'=\w+\]',s) is None:
-        return s
-
-    ret=re.sub(r"=(\w+)]", r'="\1"]', s)
-    return ret
-
+from linklist import LinkList
+from news import News
 
 def parseSource(s):
     sel=None