article source name

db.py

@@ -31,22 +31,22 @@ class db():
         if db_filter["scrap_date"]=={}:
             del db_filter["scrap_date"]
 
-        if "minweight" in filter and filter != "":
+        if "site" in filter and filter["site"] != "":
+            db_filter["source_name"]={"$regex":".*%s.*"%filter["site"], "$options": 'i'}
+
+        if "minweight" in filter and filter["minweight"] != "":
             db_filter["weight"]={"$gt":int(filter["minweight"])}
 
         if "keyword" in filter and len(filter["keyword"]) >0:
             db_filter["kw"]={"$all": filter["keyword"]}
 
-        #print(db_filter)
+        print(db_filter)
         q = self.c_articles.find(db_filter,{"html":0,"text":0}).limit(limit)
         count = q.count()
         return { 'articles': [ self.jsonable(p) for p in q ],
                  'count' : count }
 
-    def articles(self, id=None, limit=100):
-        if id is None:
-            return [ self.jsonable(p) for p in self.c_articles.find({"matches":True},{"html":0,"text":0}).limit(limit) ]
-
+    def article(self, id):
         ret=self.c_articles.find_one({"_id":ObjectId(id)})
         return self.jsonable(ret)
 

news.py

@@ -18,15 +18,17 @@ class News():
     HTML=""
     ERROR=False
     SOURCE=""
+    SOURCENAME=""
     WEIGHT_SUM=0
     MATCHES=False
     MATCHING_KW=[]
     TITLE=""
 
-    def __init__(self,url,source,kw,lang="en"):
+    def __init__(self,url,sourceid, sourcename,kw,lang="en"):
         url=sanitizeUrl(url)
         self.URL=url
-        self.SOURCE=source
+        self.SOURCENAME=sourcename
+        self.SOURCE=sourceid
         self.MATCHING_KW=[]
 
         a = Article(url, language=lang,keep_article_html=True,request_timeout=10)
@@ -63,6 +65,7 @@ class News():
         return {
                 "url": self.URL,
                 "source": self.SOURCE,
+                "source_name": self.SOURCENAME,
                 "authors": self.AUTHORS,
                 "scrap_date": self.SCRAP_DATE,
                 "publish_date": self.PUBLISH_DATE,

parser.py

@@ -30,7 +30,7 @@ def parseSource(s):
     if not q.empty():
         threads=[]
         for i in range(num_worker_threads):
-            t = threading.Thread(target=news_worker,args=[s["_id"],kw])
+            t = threading.Thread(target=news_worker,args=[s["_id"],s["name"],kw])
             t.daemon=True
             t.start()
             threads.append(t)
@@ -46,7 +46,7 @@ def parseSource(s):
             t.join(10)
     print("[Source] %s => Finished parsing" %s["name"])
 
-def news_worker(source_id,kw):
+def news_worker(source_id,source_name,kw):
     tname=threading.current_thread().name
     while True:
         try:
@@ -61,7 +61,7 @@ def news_worker(source_id,kw):
 
         print("[Thread %s] %s" % (tname,item))
         try:
-            n=News(item,source_id,kw)
+            n=News(item,source_id,source_name,kw)
             print("[Thread %s] Finished" % tname)
             d.insert_article(n.get())
         except:

web.py

@@ -54,11 +54,11 @@ def get_keywords():
 @app.route(BASEPATH+'/articles/', methods=['GET'])
 @app.route(BASEPATH+'/articles', methods=['GET'])
 def get_articles():
-    return jsonify({'articles': d.articles(limit=80)}) #limit
+    return jsonify(d.filter_articles({},limit=15)) 
 
 @app.route(BASEPATH+'/article/<string:art_id>', methods=['GET'])
 def get_article(art_id):
-    art=d.articles(id=art_id)
+    art=d.article(art_id)
     if art is None:
         abort(401)
     return jsonify({'articles': art})