analytics

analytics.py

@@ -1,8 +1,18 @@
 import requests
 
 
-def post_data(key, value):
+def post_data(field, keys, value):
+    db = 'news'
+    influx_host = 'http://localhost:8086/write?db=%s' % db
+    headers = {'Content-Type': 'application/octet-stream'}
+    tmpkeys = ",".join(["%s=%s" % (k, v) for k, v in keys.items()])
+    data = "%s,%s %s" % (field, tmpkeys, value)
+    print(data)
     try:
-        r = requests.post()
+        ret = requests.post(influx_host, data=data, headers=headers)
+        if ret.status_code // 100 != 2:
+            print(ret.status_code)
+            print(ret.text)
+            print(data, "failed!!")
     except Exception as e:
         pass

parser.py

@@ -11,6 +11,7 @@ from time import strftime,time
 
 from multiprocessing import Pool
 import sys
+from analytics import post
 
 
 def parseSource(s):
@@ -106,6 +107,8 @@ def news_worker(source, kw, d, q):
 
 
 # TODO influx start
+
+post('start', {}, 1)
 start = time()
 sources = db().sources()
 db().purge_error()
@@ -113,6 +116,11 @@ pool = Pool(processes=4)
 ret = pool.map(parseSource, sources)
 new = sum([r[3] for r in ret])
 tot = sum([r[1] for r in ret])
+
+post('new', {}, new)
+post('total', {}, tot)
+# TODO influx source new
+# TODO influx source total
 # n = sum(ret)
 # sret = sorted(ret, key=lambda tup: tup[0])
 # print(sret)
@@ -121,3 +129,4 @@ print("[%s] %d/%d new articles, total time %d sec" %
       (strftime("%H:%M:%S"), new, tot, time()-start))
 print("#" * 75)
 # TODO influx end
+post('start', {}, 0)