match by string not regex

db.py

@@ -49,7 +49,12 @@ class db():
             return False
         return True 
 
+    def lower_arr(self,arr):
+        return [a.lower() for a in arr]
     def get_keywords(self,category):
         kw=self.client[self.DB]["keywords"]
-        ret = [ {"weight":k["weight"], "kw": k["keyword"], "pattern": re.compile(k["keyword"]) } for k in  kw.find({}) ]#FIXME, categories
+        #FIXME, split on "|" or regex
+        ret = [ {"weight":k["weight"],
+                "kw": k["keyword"],
+                "pattern": self.lower_arr(k["keyword"].split("|")) } for k in  kw.find({}) ]#FIXME, categories
         return ret

news.py

@@ -31,13 +31,11 @@ class News():
         try:
             a.download()
             a.parse()
-            return
         except Exception:
             print("ERROR parsing/downloading article")
             ERROR=True
             return
         a.nlp()
-        
         a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
         
         self.SCRAP_DATE=datetime.datetime.now()
@@ -46,8 +44,11 @@ class News():
         self.SUMMARY=a.summary
         self.TEXT=a.text
         self.HTML=htmlmin.minify(a.article_html,remove_empty_space=True)
+        lowertext=self.TEXT.lower()
         for k in kw:
-            if re.match(k["pattern"], self.TEXT):
+            #if re.match(k["pattern"], self.TEXT): #FIXME regex vs string in string
+            matches=len( [ 1 for kwp in k["pattern"] if kwp in lowertext ] )
+            if matches==len(k["pattern"]):
                 self.WEIGHT_SUM+=k["weight"]
                 self.MATCHING_KW.append(k["kw"])
                 self.MATCHES=True

parser.py

@@ -52,6 +52,6 @@ def news_worker(source_id,kw):
 
 q=Queue()
 d=db()
-num_worker_threads=20
+num_worker_threads=10
 for s in d.sources():
     parseSource(s)