|
|
@@ -33,12 +33,14 @@ def parseSource(s):
|
|
|
t.start()
|
|
|
threads.append(t)
|
|
|
|
|
|
- print("Waiting for %d queue elements" % q.qsize())
|
|
|
+ print("Waiting for %d queue elements (0 if num_workers > new_articles)" % q.qsize())
|
|
|
q.join() #block until all tasks are done
|
|
|
+ print("Finished waiting for the queue")
|
|
|
|
|
|
#stop workers
|
|
|
for i in range(num_worker_threads):
|
|
|
q.put(None)
|
|
|
+ print("Waiting for %d threads" % len(threads))
|
|
|
for t in threads:
|
|
|
t.join()
|
|
|
print("[Source] %s => Finished parsing" %s["name"])
|
|
|
@@ -48,14 +50,15 @@ def news_worker(source_id,kw):
|
|
|
item = q.get()
|
|
|
if item is None:
|
|
|
break
|
|
|
+ print("[Thread %s] %s" % (threading.current_thread().name,item))
|
|
|
n=News(item,source_id,kw)
|
|
|
- print("[Thread %s] %s" % (threading.current_thread().name,n.URL))
|
|
|
+ print("[Thread %s] Finished" % threading.current_thread().name)
|
|
|
d.insert_article(n.get())
|
|
|
q.task_done()
|
|
|
|
|
|
q=Queue()
|
|
|
d=db()
|
|
|
d.purge_error()
|
|
|
-num_worker_threads=100
|
|
|
+num_worker_threads=10
|
|
|
for s in d.sources():
|
|
|
parseSource(s)
|
