|
|
@@ -14,7 +14,7 @@ def parseSource(s):
|
|
|
if "selector" in s:
|
|
|
sel=s["selector"]
|
|
|
if "rss" in s:
|
|
|
- rss=bool(["rss"])
|
|
|
+ rss=bool(s["rss"])
|
|
|
l=LinkList(s["link"],sel,rss=rss)
|
|
|
|
|
|
kw=d.get_keywords(s["category"])
|
|
|
@@ -28,7 +28,6 @@ def parseSource(s):
|
|
|
|
|
|
num_worker_threads=min(10,q.qsize())
|
|
|
if not q.empty():
|
|
|
- print("[Source] Num workers: %d" % num_worker_threads)
|
|
|
threads=[]
|
|
|
for i in range(num_worker_threads):
|
|
|
t = threading.Thread(target=news_worker,args=[s["_id"],kw])
|
|
|
@@ -36,9 +35,10 @@ def parseSource(s):
|
|
|
t.start()
|
|
|
threads.append(t)
|
|
|
|
|
|
- print("Waiting for %d queue elements (0 if num_workers > new_articles)" % q.qsize())
|
|
|
- q.join() #block until all tasks are done
|
|
|
- print("Finished waiting for the queue")
|
|
|
+ if (q.qsize() > 0):
|
|
|
+ print("Waiting for %d queue elements (0 if num_workers > new_articles)" % q.qsize())
|
|
|
+ q.join() #block until all tasks are done
|
|
|
+ print("Finished waiting for the queue")
|
|
|
|
|
|
#stop workers
|
|
|
for i in range(num_worker_threads):
|
