limit workers to new articles

parser.py

@@ -8,6 +8,7 @@ from linklist import LinkList
 from news import News
 
 def parseSource(s):
+    num_worker_threads=10
     sel=None
     rss=False
     if "selector" in s:
@@ -25,7 +26,9 @@ def parseSource(s):
         print("[Source] %s: %d new articles" % ( s["name"], q.qsize()))
 
 
+    num_worker_threads=min(10,q.qsize())
     if not q.empty():
+        print("[Source] Num workers: %d" % num_worker_threads)
         threads=[]
         for i in range(num_worker_threads):
             t = threading.Thread(target=news_worker,args=[s["_id"],kw])
@@ -42,7 +45,7 @@ def parseSource(s):
             q.put(None)
         print("Waiting for %d threads" % len(threads))
         for t in threads:
-            t.join()
+            t.join(10)
     print("[Source] %s => Finished parsing" %s["name"])
 
 def news_worker(source_id,kw):
@@ -51,14 +54,17 @@ def news_worker(source_id,kw):
         if item is None:
             break
         print("[Thread %s] %s" % (threading.current_thread().name,item))
-        n=News(item,source_id,kw)
-        print("[Thread %s] Finished" % threading.current_thread().name)
-        d.insert_article(n.get())
-        q.task_done()
+        try:
+            n=News(item,source_id,kw)
+            print("[Thread %s] Finished" % threading.current_thread().name)
+            d.insert_article(n.get())
+        except:
+            print("whut")
+        finally:
+            q.task_done()
 
 q=Queue()
 d=db()
 d.purge_error()
-num_worker_threads=10
 for s in d.sources():
     parseSource(s)