|
|
@@ -24,6 +24,7 @@ def parseSource(s):
|
|
|
l = LinkList(s["link"], sel, rss=rss)
|
|
|
|
|
|
kw = d.get_keywords(s["category"])
|
|
|
+ new_articles = 0
|
|
|
# print(kw)
|
|
|
if l is not None:
|
|
|
for a in l.links:
|
|
|
@@ -31,6 +32,7 @@ def parseSource(s):
|
|
|
q.put(a)
|
|
|
print("[%s][Source] %s: %d/%d new/total articles" %
|
|
|
(strftime("%H:%M:%S"), s["name"], q.qsize(), len(l.links)))
|
|
|
+ new_articles = q.qsize()
|
|
|
|
|
|
if not q.empty():
|
|
|
num_worker_threads = min(10, q.qsize())
|
|
|
@@ -50,6 +52,7 @@ def parseSource(s):
|
|
|
for t in threads:
|
|
|
t.join(10)
|
|
|
# print("[Source] %s => Finished parsing" %s["name"])
|
|
|
+ return new_articles
|
|
|
|
|
|
|
|
|
def news_worker(source, kw):
|
|
|
@@ -96,8 +99,10 @@ def news_worker(source, kw):
|
|
|
q = Queue()
|
|
|
d = db()
|
|
|
d.purge_error()
|
|
|
+n = 0
|
|
|
for s in d.sources():
|
|
|
- parseSource(s)
|
|
|
+ n = n + parseSource(s)
|
|
|
sys.stdout.flush()
|
|
|
|
|
|
+print("[%s] %d new articles" % (strftime("%H:%M:%S"), n))
|
|
|
print("###############################################################################")
|
