keywords

linklist.py

@@ -11,7 +11,6 @@ class LinkList():
                 print("RSS ERROR. PANIC")
             #print(feed["bozo"]) FIXME: If bozo==1 => error
             self.links=[i["link"] for i in feed["items"]]
-            print(self.links)
         else:
             if selector is None:
                 #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida

news.py

@@ -26,6 +26,7 @@ class News():
         url=sanitizeUrl(url)
         self.URL=url
         self.SOURCE=source
+        self.MATCHING_KW=[]
 
         a = Article(url, language=lang,keep_article_html=True)
         try:
@@ -52,7 +53,6 @@ class News():
                 self.WEIGHT_SUM+=k["weight"]
                 self.MATCHING_KW.append(k["kw"])
                 self.MATCHES=True
-            
 
     def get(self):
         return {

parser.py

@@ -33,7 +33,10 @@ def parseSource(s):
             t.start()
             threads.append(t)
     
-        q.join()
+        print("Waiting for %d queue elements" % q.qsize())
+        q.join() #block until all tasks are done
+
+        #stop workers
         for i in range(num_worker_threads):
             q.put(None)
         for t in threads:
@@ -52,6 +55,6 @@ def news_worker(source_id,kw):
 
 q=Queue()
 d=db()
-num_worker_threads=10
+num_worker_threads=100
 for s in d.sources():
     parseSource(s)