? misc db changes?

db.py

@@ -4,6 +4,7 @@ import datetime
 import re
 import news
 import pytz
+import time
 
 
 class db():
@@ -15,11 +16,12 @@ class db():
         self.c_articles = self.client[self.DB]["articles"]
 
     def change_timezone(self, el):
-        gmt = pytz.timezone("America/Buenos_Aires")  # FIXME
+        #gmt = pytz.timezone("America/Buenos_Aires")  # FIXME
+        local = pytz.timezone("US/Central")
         est = pytz.timezone('US/Eastern')
         fmt = '%Y-%m-%d %H:%M:%S %Z%z'
 
-        el["scrap_date"] = gmt.localize(el["scrap_date"])
+        el["scrap_date"] = local.localize(el["scrap_date"])
         el["scrap_date"] = el["scrap_date"].astimezone(est)
         el["scrap_date"] = el["scrap_date"].strftime(fmt)
         return el
@@ -32,6 +34,7 @@ class db():
         return el
 
     def filter_articles(self, filter, limit=80):
+        start = time.time()
         db_filter = {}
         db_filter["matches"] = True
         db_filter["deleted"] = False
@@ -83,15 +86,24 @@ class db():
         if "page" in filter:
             pageNumber = filter["page"]
         # print(db_filter)
-        res = self.c_articles.find(db_filter, {"html": 0, "text": 0})
+        print(db_filter)
+        fields = {"html": 0, "text": 0, "authors":0,"lowerurl":0}
+        fields = {"kw":1,"scrap_date":1, "source_name":1,"summary":1,"title":1,"url":1,"weight":1}
+        res = self.c_articles.find(db_filter, fields)
         if res is not None:
             toSkip = (((pageNumber - 1) * limit) if pageNumber > 0 else 0)
-            q = res.skip(toSkip).sort("scrap_date", DESCENDING).limit(limit)
-            count = q.count()
+            #q = res.skip(toSkip).sort("scrap_date", DESCENDING).limit(limit)
+            q = res.skip(toSkip).sort("scrap_date", DESCENDING).limit(limit*3) #+2 pages
+            start=time.time()
         else:
             q = []
             count = 0
-        return {'articles': [self.jsonable(self.change_timezone(p)) for p in q],
+        ret_arts = [self.jsonable(self.change_timezone(p)) for p in q]
+        count = len(ret_arts)+toSkip
+        ret_arts = ret_arts[:limit]
+        #count = len(ret_arts)
+        print("pre-ret", time.time()-start)
+        return {'articles': ret_arts,
                 'count': count}
 
     def article(self, id):
@@ -125,11 +137,15 @@ class db():
     def purge_error(self):
         self.c_articles.remove({'error': True})
 
-    def article_exists(self, url):
-        ret = self.c_articles.find_one({'lowerurl': url.lower()})
-        if ret is None:
-            return False
-        return True
+    def article_exists(self, url, src):
+	#slow as balls
+        #ret = self.c_articles.find({'source': src, 'lowerurl': url.lower()},{"_id":1}).limit(1) #faster than findone
+        #articles may have different src and same url.
+
+        ret = self.c_articles.find({'lowerurl': url.lower()},{"_id":1}).limit(1) #faster than findone
+        c = len([ 1 for r in ret ] )
+	#findOne returns the elem. find returns the cursor (pointer)
+        return c>0
 
     def lower_arr(self, arr):
         return [a.lower() for a in arr]