misc debug

news.py

@@ -49,7 +49,10 @@ class News():
         a = None
         try:
             a = Article(url=url, config=config)
-            a.download(html=html)
+            if html is not None:
+                a.download(html=html)
+            else:
+                a.download()
 
             a.parse()
             a.nlp()

parser.py

@@ -104,28 +104,29 @@ def news_worker(source, kw, d, q):
             q.task_done()
 
 
-post_data('event', {}, '"start"')
-start = time()
-print("init db")
-d = db()
-sources = d.sources()
-post_data('value', {'type': 'error'}, d.count_error())
-d.purge_error()
-
-pool = Pool(processes=4)
-ret = pool.map(parseSource, sources)
-new = sum([r[3] for r in ret])
-tot = sum([r[1] for r in ret])
-
-post_data('value', {'type': 'new'}, new)
-post_data('value', {'type': 'total'}, tot)
-# n = sum(ret)
-# sret = sorted(ret, key=lambda tup: tup[0])
-# print(sret)
-
-print("[%s] %d/%d new articles, total time %d sec" %
-      (strftime("%H:%M:%S"), new, tot, time()-start))
-post_data('time', {'type': 'total'}, time()-start)
-print("#" * 75)
-post_data('value', {'type': 'error'}, d.count_error())
-post_data('event', {}, '"end"')
+if __name__ == '__main__':
+    post_data('event', {}, '"start"')
+    start = time()
+    print("init db")
+    d = db()
+    sources = d.sources()
+    post_data('value', {'type': 'error'}, d.count_error())
+    d.purge_error()
+    
+    pool = Pool(processes=4)
+    ret = pool.map(parseSource, sources)
+    new = sum([r[3] for r in ret])
+    tot = sum([r[1] for r in ret])
+    
+    post_data('value', {'type': 'new'}, new)
+    post_data('value', {'type': 'total'}, tot)
+    # n = sum(ret)
+    # sret = sorted(ret, key=lambda tup: tup[0])
+    # print(sret)
+    
+    print("[%s] %d/%d new articles, total time %d sec" %
+          (strftime("%H:%M:%S"), new, tot, time()-start))
+    post_data('time', {'type': 'total'}, time()-start)
+    print("#" * 75)
+    post_data('value', {'type': 'error'}, d.count_error())
+    post_data('event', {}, '"end"')