add kw analytics

analytics.py

@@ -36,7 +36,11 @@ if __name__ == '__main__':
         val = e['_id'].replace(" ", r"\ ")
         post_data('error', {'source': '%s' % val}, e['count'])
 
-    for d in d.get_sources_with_count():
-        val = d['_id'].replace(" ", r"\ ")
+    for s in d.get_sources_with_count():
+        val = s['_id'].replace(" ", r"\ ")
         post_data('totals', {'source': '%s' % val},
-                  {'value': d['count'], 'weight': d['weight']})
+                  {'value': s['count'], 'weight': s['weight']})
+
+    for k, v in d.get_keywords_with_count().items():
+        val = k.replace(" ", r"\ ")
+        post_data('totals', {}, {'kw': val, 'count': v})

db.py

@@ -1,5 +1,7 @@
 from pymongo import MongoClient, errors, DESCENDING, ASCENDING
 from bson.objectid import ObjectId
+from datetime import datetime, timedelta
+
 import datetime
 import re
 import news
@@ -202,3 +204,18 @@ class db():
 
         # t = self.c_articles.aggregate([{'$group': {'_id': '$source_name', 'count': {'$sum': 1}}}])
         return r
+
+    def get_keywords_with_count(self):
+        this_year = datetime.now() - timedelta(days=180)
+        r = self.c_articles.aggregate([{'$match': {
+                                            '$and': [{'matches': True}, {'scrap_date': {'$gt': this_year}}]
+                                        }},
+                                       {'$unwind': '$kw'},
+                                       {'$group':
+                                           {'_id': '$kw', 'sum': {'$sum': 1}}
+                                        }
+                                       ])
+        d1 = {}
+        for res in r:
+            d1[res['_id']] = res['sum']
+        return d1