from pymongo import MongoClient, errors, DESCENDING, ASCENDING from bson.objectid import ObjectId import datetime import re import news import pytz import time class db(): DB = "alexis" def __init__(self): self.client = MongoClient('localhost', 27017) self.c_sources = self.client[self.DB]["sources"] self.c_articles = self.client[self.DB]["articles"] def change_timezone(self, el): #gmt = pytz.timezone("America/Buenos_Aires") # FIXME local = pytz.timezone("US/Central") est = pytz.timezone('US/Eastern') fmt = '%Y-%m-%d %H:%M:%S %Z%z' el["scrap_date"] = local.localize(el["scrap_date"]) el["scrap_date"] = el["scrap_date"].astimezone(est) el["scrap_date"] = el["scrap_date"].strftime(fmt) return el def jsonable(self, el): if el is None or "_id" not in el: return el el["_id"] = str(el["_id"]) return el def filter_articles(self, filter, limit=80): start = time.time() db_filter = {} db_filter["matches"] = True db_filter["deleted"] = False db_filter["scrap_date"] = {} if "maxdate" in filter and len(filter["maxdate"]) > 7: db_filter["scrap_date"]["$lt"] = datetime.datetime.strptime( filter["maxdate"], "%Y-%m-%d") if "mindate" in filter and len(filter["mindate"]) > 7: db_filter["scrap_date"]["$gt"] = datetime.datetime.strptime( filter["mindate"], "%Y-%m-%d") if db_filter["scrap_date"] == {}: del db_filter["scrap_date"] if "title" in filter and filter["title"] != "": filter["title"] = filter["title"].replace( "(", "\(").replace(")", "\)") db_filter["title"] = {"$regex": ".*%s.*" % filter["title"], "$options": 'i'} if "site" in filter and filter["site"] != "": filter["site"] = filter["site"].replace( "(", "\(").replace(")", "\)") db_filter["source_name"] = { "$regex": ".*%s.*" % filter["site"], "$options": 'i'} # db_filter["$or"]=[ # {"source_name": {"$regex":".*%s.*"%filter["site"], "$options": 'i'}}, # {"url": {"$regex":".*%s.*"%filter["site"], "$options": 'i' }} # ] if "minweight" not in filter or not filter["minweight"].isdigit(): filter["minweight"] = 1 if "category" not in filter or filter["category"] == "-1": filter["category"] = -1 if int(filter["category"]) >= 0: db_filter["category"] = int(filter["category"]) # pass db_filter["weight"] = {"$gt": int(filter["minweight"])} if "keyword" in filter and len(filter["keyword"]) > 0: db_filter["kw"] = {"$all": filter["keyword"]} # db_filter["kw.0"]={"$exists":False} pageNumber = 0 if "page" in filter: pageNumber = filter["page"] # print(db_filter) print(db_filter) fields = {"html": 0, "text": 0, "authors":0,"lowerurl":0} fields = {"kw":1,"scrap_date":1, "source_name":1,"summary":1,"title":1,"url":1,"weight":1} res = self.c_articles.find(db_filter, fields) if res is not None: toSkip = (((pageNumber - 1) * limit) if pageNumber > 0 else 0) #q = res.skip(toSkip).sort("scrap_date", DESCENDING).limit(limit) q = res.skip(toSkip).sort("scrap_date", DESCENDING).limit(limit*3) #+2 pages start=time.time() else: q = [] count = 0 ret_arts = [self.jsonable(self.change_timezone(p)) for p in q] count = len(ret_arts)+toSkip ret_arts = ret_arts[:limit] #count = len(ret_arts) print("pre-ret", time.time()-start) return {'articles': ret_arts, 'count': count} def article(self, id): ret = self.c_articles.find_one({"_id": ObjectId(id)}) return self.jsonable(ret) def delete_article(self, artid): return self.c_articles.update({"_id": ObjectId(artid)}, {"$set": {"deleted": True}}) def undelete_article(self, artid): return self.c_articles.update({"_id": ObjectId(artid)}, {"$set": {"deleted": False}}) def sources(self, id=None): if id is None: return [self.jsonable(p) for p in self.c_sources.find()] ret = self.c_sources.find_one({"_id": ObjectId(id)}) return self.jsonable(ret) def rss_sources(self): return [p for p in self.c_sources.find({"rss": 1})] def insert_article(self, o): try: self.c_articles.insert_one(o) return True except errors.DuplicateKeyError: print("Dup") return False def count_error(self): return self.c_articles.count({'error': True}) def purge_error(self): self.c_articles.remove({'error': True}) def article_exists(self, url, src): #slow as balls #ret = self.c_articles.find({'source': src, 'lowerurl': url.lower()},{"_id":1}).limit(1) #faster than findone #articles may have different src and same url. ret = self.c_articles.find({'lowerurl': url.lower()},{"_id":1}).limit(1) #faster than findone c = len([ 1 for r in ret ] ) #findOne returns the elem. find returns the cursor (pointer) return c>0 def lower_arr(self, arr): return [a.lower() for a in arr] def to_regex(self, arr): # "ACH" | "PAYMENT" | "EBAY" return [re.compile('\\b' + w.lower() + '\\b') for w in arr] def get_keywords_list(self): kw = self.client[self.DB]["keywords"] return [k["keyword"] for k in kw.find({})] def get_keywords(self, category): kw = self.client[self.DB]["keywords"] # FIXME, split on "|" or regex ret = [{"weight": k["weight"], "kw": k["keyword"], "pattern": self.to_regex(k["keyword"].split("|"))} for k in kw.find({})] # FIXME, categories return ret def add_source(self, url, rss, selector, name, category): self.c_sources.insert({"link": url, "rss": rss, "selector": selector, "name": name, "category": category}) def reparse_articles(self, articles, kw): for a in articles: # print(a) art = self.c_articles.find_one({"url": a["url"]}) res = news.match_keywords(kw, art["text"]) #(matches,w_sum,matching) # if(res["matches"]): print(res) self.c_articles.update({"_id": ObjectId(a["_id"])}, {"$set": res}) #matches, kw, weight def get_sources_with_error(self): r = self.c_articles.aggregate([{'$match': {'error': True}}, {'$group': {'_id': '$source_name', 'count': {'$sum': 1}}}]) return r def get_sources_with_count(self): r = self.c_articles.aggregate([{'$match': {'matches': True}}, {'$group': {'_id': '$source_name', 'count': {'$sum': 1}, 'weight': {'$avg': '$weight'} } }]) # t = self.c_articles.aggregate([{'$group': {'_id': '$source_name', 'count': {'$sum': 1}}}]) return r