|
@@ -9,6 +9,7 @@ from news import News
|
|
|
from paid_parser import PaidParser
|
|
from paid_parser import PaidParser
|
|
|
|
|
|
|
|
def parseSource(s):
|
|
def parseSource(s):
|
|
|
|
|
+ global q
|
|
|
q=Queue()
|
|
q=Queue()
|
|
|
num_worker_threads=10
|
|
num_worker_threads=10
|
|
|
sel=None
|
|
sel=None
|
|
@@ -26,8 +27,8 @@ def parseSource(s):
|
|
|
q.put(a)
|
|
q.put(a)
|
|
|
print("[Source] %s: %d/%d new/total articles" % ( s["name"], q.qsize(), len(l.links)))
|
|
print("[Source] %s: %d/%d new/total articles" % ( s["name"], q.qsize(), len(l.links)))
|
|
|
|
|
|
|
|
- num_worker_threads=min(10,q.qsize())
|
|
|
|
|
if not q.empty():
|
|
if not q.empty():
|
|
|
|
|
+ num_worker_threads=min(10,q.qsize())
|
|
|
threads=[]
|
|
threads=[]
|
|
|
for i in range(num_worker_threads):
|
|
for i in range(num_worker_threads):
|
|
|
t = threading.Thread(target=news_worker,args=[s,kw])
|
|
t = threading.Thread(target=news_worker,args=[s,kw])
|
|
@@ -57,7 +58,10 @@ def news_worker(source,kw):
|
|
|
try:
|
|
try:
|
|
|
item = q.get(timeout=5)
|
|
item = q.get(timeout=5)
|
|
|
except:
|
|
except:
|
|
|
|
|
+ print("Failed to get item")
|
|
|
|
|
+ q.task_done()
|
|
|
break
|
|
break
|
|
|
|
|
+
|
|
|
if item is None:
|
|
if item is None:
|
|
|
q.task_done()
|
|
q.task_done()
|
|
|
break
|
|
break
|
