alexis2/db.py

from pymongo import MongoClient,errors,DESCENDING,ASCENDING
from bson.objectid import ObjectId
import datetime
import re
import news

class db():
    DB="alexis"
    def __init__(self):
        self.client = MongoClient('localhost', 27017)
        self.c_sources=self.client[self.DB]["sources"]
        self.c_articles=self.client[self.DB]["articles"]

    def jsonable(self, el):
        if el is None or "_id" not in el:
            return el

        el["_id"]=str(el["_id"])
        return el


    def filter_articles(self, filter, limit=80):
        db_filter={}
        db_filter["matches"]=True
        db_filter["deleted"]=False
        db_filter["scrap_date"]={}
        if "maxdate" in filter and len(filter["maxdate"])>7:
            db_filter["scrap_date"]["$lt"]=datetime.datetime.strptime(filter["maxdate"],"%Y-%m-%d")

        if "mindate" in filter and len(filter["mindate"])>7:
            db_filter["scrap_date"]["$gt"]=datetime.datetime.strptime(filter["mindate"],"%Y-%m-%d")

        if db_filter["scrap_date"]=={}:
            del db_filter["scrap_date"]

        if "site" in filter and filter["site"] != "":
            filter["site"]=filter["site"].replace("(","\(").replace(")","\)")
            db_filter["source_name"]={"$regex":".*%s.*"%filter["site"], "$options": 'i'}
            #db_filter["$or"]=[
            #    {"source_name": {"$regex":".*%s.*"%filter["site"], "$options": 'i'}},
            #    {"url": {"$regex":".*%s.*"%filter["site"], "$options": 'i' }}
            #    ]

        if "minweight" in filter and filter["minweight"] != "":
            db_filter["weight"]={"$gt":int(filter["minweight"])}

        if "keyword" in filter and len(filter["keyword"]) >0:
            db_filter["kw"]={"$all": filter["keyword"]}


        #db_filter["kw.0"]={"$exists":False}
        pageNumber=0
        if "page" in filter:
            pageNumber=filter["page"]
        #print(db_filter)
        res = self.c_articles.find(db_filter,{"html":0,"text":0})
        if res is not None:
            toSkip=(((pageNumber-1)*limit) if pageNumber > 0 else 0)
            q = res.skip(toSkip).sort("scrap_date",DESCENDING).limit(limit)
            count = q.count()
        else:
            q = []
            count = 0
        return { 'articles': [ self.jsonable(p) for p in q ],
                 'count' : count }

    def article(self, id):
        ret=self.c_articles.find_one({"_id":ObjectId(id)})
        return self.jsonable(ret)

    def delete_article(self,artid):
        self.c_articles.update({"_id":ObjectId(artid)},{"$set":{"deleted":True}})
        return True

    def sources(self,id=None):
        if id is None:
            return [ self.jsonable(p) for p in self.c_sources.find() ]

        ret=self.c_sources.find_one({"_id":ObjectId(id)})
        return self.jsonable(ret)

    def rss_sources(self):
        return [ p for p in self.c_sources.find({"rss":1}) ]


    def insert_article(self,o):
        try:
            self.c_articles.insert_one(o)
            return True
        except errors.DuplicateKeyError:
            print("Dup")
            return False

    def purge_error(self):
        self.c_articles.remove({'error':True})

    def article_exists(self,url):
        ret=self.c_articles.find_one({'url':url})
        if ret is None:
            return False
        return True 

    def lower_arr(self,arr):
        return [a.lower() for a in arr]

    def to_regex(self,arr): #"ACH" |  "PAYMENT" | "EBAY"
        return [ re.compile('\\b'+w.lower()+'\\b') for w in arr ]

    def get_keywords_list(self):
        kw=self.client[self.DB]["keywords"]
        return [  k["keyword"] for k in kw.find({}) ]

    def get_keywords(self,category):
        kw=self.client[self.DB]["keywords"] #FIXME, split on "|" or regex
        ret = [ {"weight":k["weight"],
                "kw": k["keyword"],
                "pattern": self.to_regex(k["keyword"].split("|")) } for k in  kw.find({}) ]#FIXME, categories
        return ret

    def add_source(self,url,rss,selector,name):
        self.c_sources.insert({"link":url,
                                "rss":rss,
                                "selector":selector,
                                "name":name,
                                "category":0})


    def reparse_articles(self,articles,kw):
        for a in articles:
            #print(a)
            art=self.c_articles.find_one({"url":a["url"]})
            res=news.match_keywords(kw,art["text"])
            #(matches,w_sum,matching) 
            #if(res["matches"]):
            print(res)
            self.c_articles.update({"_id":ObjectId(a["_id"])}, {"$set": res })
            #matches, kw, weight