alexis2/stats.py

from pymongo import MongoClient, errors, DESCENDING, ASCENDING
from bson.objectid import ObjectId
import datetime
import re
import news
import pytz
import time
from pprint import pprint


class stats():
    DB = "alexis"

    def __init__(self):
        self.client = MongoClient('localhost', 27017)
        self.c_sources = self.client[self.DB]["sources"]
        self.c_articles = self.client[self.DB]["articles"]

    def jsonable(self, el):
        if el is None or "_id" not in el:
            return el

        el["_id"] = str(el["_id"])
        return el

    def sources_month(self, d, period):
        article_filter = {}
        article_filter['scrap_date'] = {'$gt': d - period}

        fields = {"source": 1}
        sources = [ p for p in self.c_sources.find({},{'name':1}) ]

        res = self.c_articles.find(article_filter, fields)

        ret = {}
        for s in sources:
            source = str(s['_id'])
            if source not in ret:
                ret[source] = 0

        for p in res:
            if p['source'] in ret:
                ret[p['source']] += 1
            else:
                ret[p['source']] = 1

        realret = {}
        for s in sources:
            source = str(s['_id'])
            realret[s['name']] = ret[source]

        for l in self.dict_to_csv(realret):
            print(l)

        return ret
        
    def sources(self, id=None):
        if id is None:
            return [self.jsonable(p) for p in self.c_sources.find()]

        ret = self.c_sources.find_one({"_id": ObjectId(id)})
        return self.jsonable(ret)

    def rss_sources(self):
        return [p for p in self.c_sources.find({"rss": 1})]

    def get_keywords_list(self):
        kw = self.client[self.DB]["keywords"]
        return [k["keyword"] for k in kw.find({})]

    def keywords_count(self):
        kw = {}
        for a in self.c_articles.find({'matches':True},{'kw':1}):
           if 'kw' not in a:
               print(a)
               continue
           for k in a['kw']: 
               if k in kw:
                   kw[k] += 1
               else:
                   kw[k] = 1
        return kw

    def dict_to_csv(self, kv):
        ret = []
        for key in kv:
            ret.append('"%s",%s' % (key, kv[key]))
        return ret
            


if __name__ == '__main__':
    s = stats()
    #for line in s.dict_to_csv(s.keywords_count()):
    #    print(line)
    s.sources_month(datetime.datetime.today(), datetime.timedelta(1))
    s.sources_month(datetime.datetime.today(), datetime.timedelta(2))
    #for source in s.sources():
    #    print(source['name'])