reparse

db.py

@@ -2,6 +2,7 @@ from pymongo import MongoClient,errors,DESCENDING,ASCENDING
 from bson.objectid import ObjectId
 import datetime
 import re
+import news
 
 class db():
     DB="alexis"
@@ -45,6 +46,8 @@ class db():
         if "keyword" in filter and len(filter["keyword"]) >0:
             db_filter["kw"]={"$all": filter["keyword"]}
 
+
+        #db_filter["kw.0"]={"$exists":False}
         pageNumber=0
         if "page" in filter:
             pageNumber=filter["page"]
@@ -96,7 +99,7 @@ class db():
         return [a.lower() for a in arr]
 
     def to_regex(self,arr): #"ACH" |  "PAYMENT" | "EBAY"
-        return [ re.compile('\b'+w.lower()+'\b') for w in arr ]
+        return [ re.compile('\\b'+w.lower()+'\\b') for w in arr ]
 
     def get_keywords_list(self):
         kw=self.client[self.DB]["keywords"]
@@ -115,3 +118,16 @@ class db():
                                 "selector":selector,
                                 "name":name,
                                 "category":0})
+
+
+    def reparse_articles(self,articles,kw):
+        for a in articles:
+            #print(a)
+            art=self.c_articles.find_one({"url":a["url"]})
+            res=news.match_keywords(kw,art["text"])
+            #(matches,w_sum,matching) 
+            #if(res["matches"]):
+            print(res)
+            self.c_articles.update({"_id":ObjectId(a["_id"])}, {"$set": res })
+            #matches, kw, weight
+

news.py

@@ -66,13 +66,11 @@ class News():
 #            self.HTML=htmlmin.minify(html,remove_empty_space=True)
         self.ERROR=False
         lowertext=self.TEXT.lower()
-        for k in kw:
-            #if re.match(k["pattern"], self.TEXT): #FIXME regex vs string in string
-            matches=len( [ 1 for kwp in k["pattern"] if kwp.search(lowertext) ] )
-            if matches==len(k["pattern"]):
-                self.WEIGHT_SUM+=k["weight"]
-                self.MATCHING_KW.append(k["kw"])
-                self.MATCHES=True
+        ret=match_keywords(kw,lowertext)
+        self.MATCHES=ret["matches"]
+        self.WEIGHT_SUM=ret["weight"]
+        self.MATCHING_KW=ret["kw"]
+
     def get(self):
         return {
                 "url": self.URL,
@@ -107,3 +105,16 @@ class News():
         return a.replace("_", " ").lower()
 
 
+def match_keywords(keywords, lowertext):
+    w_sum=0
+    matching=[]
+    matches=False
+    for k in keywords:
+        num_matches=len( [ 1 for kwp in k["pattern"] if kwp.search(lowertext) ] )
+        if num_matches==len(k["pattern"]):
+            w_sum+=k["weight"]
+            matching.append(k["kw"])
+            matches=True
+
+    ret=({"matches":matches,"weight":w_sum,"kw":matching})
+    return ret

reparse.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+from db import db
+
+d=db()
+kw=d.get_keywords(0)
+
+arts=d.filter_articles({"kw.0":{"$exists":False}},limit=8000)
+#print(arts)
+print(len(arts["articles"]))
+d.reparse_articles(arts["articles"],kw)