parsed + stdout flush + log dates

parser.py

@@ -1,44 +1,48 @@
-#!/usr/bin/python3 
+#!/usr/bin/python3
 import threading
-from queue import Queue 
+from queue import Queue
 
 from db import db
 from source import Source
 from linklist import LinkList
 from news import News
 from paid_parser import PaidParser
+from time import strftime
+import sys
+
 
 def parseSource(s):
     global q
-    q=Queue()
-    num_worker_threads=10
-    sel=None
-    rss=False
+    q = Queue()
+    num_worker_threads = 10
+    sel = None
+    rss = False
     if "selector" in s:
-        sel=s["selector"]
+        sel = s["selector"]
     if "rss" in s:
-        rss=bool(s["rss"])
-    l=LinkList(s["link"],sel,rss=rss)
+        rss = bool(s["rss"])
+    l = LinkList(s["link"], sel, rss=rss)
 
-    kw=d.get_keywords(s["category"])
+    kw = d.get_keywords(s["category"])
 #    print(kw)
     if l is not None:
         for a in l.links:
             if not d.article_exists(a.lower()):
                 q.put(a)
-        print("[Source] %s: %d/%d new/total articles" % ( s["name"], q.qsize(), len(l.links)))
+        print("[%s][Source] %s: %d/%d new/total articles" %
+              (strftime("%H:%M:%S"), s["name"], q.qsize(), len(l.links)))
 
     if not q.empty():
-        num_worker_threads=min(10,q.qsize())
-        threads=[]
+        num_worker_threads = min(10, q.qsize())
+        threads = []
         for i in range(num_worker_threads):
-            t = threading.Thread(target=news_worker,args=[s,kw])
-            t.daemon=True
+            t = threading.Thread(target=news_worker, args=[s, kw])
+            t.daemon = True
             t.start()
             threads.append(t)
-    
+
 #        print("Waiting for %d queue elements" % q.qsize())
-        q.join() #block until all tasks are done
+        q.join()  # block until all tasks are done
 #        print("Finished waiting for the queue")
 
         for i in range(num_worker_threads):
@@ -47,15 +51,16 @@ def parseSource(s):
             t.join(10)
 #    print("[Source] %s => Finished parsing" %s["name"])
 
-def news_worker(source,kw):
-    source_id=source["_id"]
-    source_name=source["name"]
-    source_category=source["category"]
-    paid=False
+
+def news_worker(source, kw):
+    source_id = source["_id"]
+    source_name = source["name"]
+    source_category = source["category"]
+    paid = False
     if "paid" in source and source["paid"]:
-        paid=True
+        paid = True
 
-    tname=threading.current_thread().name
+    tname = threading.current_thread().name
     while True:
         try:
             item = q.get(timeout=5)
@@ -70,12 +75,13 @@ def news_worker(source,kw):
 
         #print("[Thread %s] %s" % (tname,item))
         try:
-            html=None
+            html = None
             if paid:
-                p=PaidParser(item)
-                html=p.html
+                p = PaidParser(item)
+                html = p.html
 
-            n=News(item,source_id,source_name,source_category,kw,html=html)
+            n = News(item, source_id, source_name,
+                     source_category, kw, html=html)
             #print("[Thread %s] Finished" % tname)
             d.insert_article(n.get())
         except Exception as e:
@@ -87,10 +93,11 @@ def news_worker(source,kw):
         finally:
             q.task_done()
 
-q=Queue()
-d=db()
+q = Queue()
+d = db()
 d.purge_error()
 for s in d.sources():
     parseSource(s)
+    sys.stdout.flush()
 
 print("###############################################################################")