multithread v1

parser.py

@@ -6,6 +6,9 @@ import pymongo
 import htmlmin
 import feedparser
 import datetime
+import threading
+
+from queue import Queue 
 from frontpage import FrontPage
 from newspaper import Article
 from db import db
@@ -115,9 +118,8 @@ def sanitizeSelector(s):
     ret=re.sub(r"=(\w+)]", r'="\1"]', s)
     return ret
 
-d=db()
-for s in d.sources():
-    print(s["name"])
+
+def parseSource(s):
     sel=None
     rss=False
     if "selector" in s:
@@ -130,7 +132,39 @@ for s in d.sources():
         for a in l.links:
             if not d.article_exists(a):
                 print(a)
-                n=News(a,s["_id"])
-                d.insert_article(n.get())
+                q.put(a)
         print(len(l.links))
-        break
+
+    num_worker_threads=8
+    threads=[]
+    for i in range(num_worker_threads):
+        t = threading.Thread(target=news_worker,args=[s["_id"]])
+        t.daemon=True
+        t.start()
+        threads.append(t)
+
+    q.join()
+    for i in range(num_worker_threads):
+        q.put(None)
+    for t in threads:
+        t.join()
+    print("Finished parsing %s" %s)
+
+
+
+def news_worker(source_id):
+    while True:
+        item = q.get()
+        if item is None:
+            break
+        n=News(item,source_id)
+        print("[Thread %s] %s" % (threading.current_thread(),n.URL))
+        d.insert_article(n.get())
+        q.task_done()
+
+q=Queue()
+d=db()
+for s in d.sources():
+    print(s["name"])
+    parseSource(s)
+    break