keyword support

db.py

@@ -1,11 +1,13 @@
 from pymongo import MongoClient,errors
 from bson.objectid import ObjectId
+import re
 
 class db():
+    DB="alexis"
     def __init__(self):
         self.client = MongoClient('localhost', 27017)
-        self.c_sources=self.client.alexis["sources"]
-        self.c_articles=self.client.alexis["articles"]
+        self.c_sources=self.client[self.DB]["sources"]
+        self.c_articles=self.client[self.DB]["articles"]
 
     def jsonable(self, el):
         if el is None or "_id" not in el:
@@ -46,3 +48,8 @@ class db():
         if ret is None:
             return False
         return True 
+
+    def get_keywords(self,category):
+        kw=self.client[self.DB]["keywords"]
+        ret = [ {"weight":k["weight"], "kw": k["keyword"], "pattern": re.compile(k["keyword"]) } for k in  kw.find({}) ]#FIXME, categories
+        return ret

parser.py

@@ -58,20 +58,27 @@ class News():
     HTML=""
     ERROR=False
     SOURCE=""
+    WEIGHT_SUM=0
+    MATCHES=False
+    MATCHING_KW=[]
 
-    def __init__(self,url,source,lang="en"):
+    def __init__(self,url,source,kw,lang="en"):
         url=sanitizeUrl(url)
         self.URL=url
         self.SOURCE=source
 
         a = Article(url, language=lang,keep_article_html=True)
-        a.download()
         try:
+            a.download()
             a.parse()
         except newspaper.article.ArticleException:
             print("ERROR parsing article")
             ERROR=True
             return
+        except Exception:
+            print("ERROR downloading article")
+            ERROR=True
+            return
         a.nlp()
         
         a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
@@ -82,6 +89,12 @@ class News():
         self.SUMMARY=a.summary
         self.TEXT=a.text
         self.HTML=htmlmin.minify(a.article_html,remove_empty_space=True)
+        for k in kw:
+            if re.match(k["pattern"], self.TEXT):
+                self.WEIGHT_SUM+=k["weight"]
+                self.MATCHING_KW.append(k["kw"])
+                self.MATCHES=True
+            
 
     def get(self):
         return {
@@ -92,6 +105,9 @@ class News():
                 "publish_date": self.PUBLISH_DATE,
                 "summary": self.SUMMARY,
                 "text": self.TEXT,
+                "matches": self.MATCHES,
+                "weight": self.WEIGHT_SUM,
+                "kw": self.MATCHING_KW,
                 "html": self.HTML 
                 }
 
@@ -135,6 +151,7 @@ def parseSource(s):
         rss=bool(["rss"])
     l=LinkList(s["link"],sel,rss=rss)
 
+    kw=d.get_keywords(s["category"])
     if l is not None:
         print("[Source] %s: %d total articles" % ( s["name"], len(l.links)))
         for a in l.links:
@@ -146,7 +163,7 @@ def parseSource(s):
     if not q.empty():
         threads=[]
         for i in range(num_worker_threads):
-            t = threading.Thread(target=news_worker,args=[s["_id"]])
+            t = threading.Thread(target=news_worker,args=[s["_id"],kw])
             t.daemon=True
             t.start()
             threads.append(t)
@@ -158,12 +175,12 @@ def parseSource(s):
             t.join()
     print("[Source] %s => Finished parsing" %s["name"])
 
-def news_worker(source_id):
+def news_worker(source_id,kw):
     while True:
         item = q.get()
         if item is None:
             break
-        n=News(item,source_id)
+        n=News(item,source_id,kw)
         print("[Thread %s] %s" % (threading.current_thread().name,n.URL))
         d.insert_article(n.get())
         q.task_done()