Merge branch 'master' of ssh://gogs.davidventura.com.ar:443/david/alexis2

db.py

@@ -40,8 +40,8 @@ class db():
         if "keyword" in filter and len(filter["keyword"]) >0:
             db_filter["kw"]={"$all": filter["keyword"]}
 
-        print(db_filter)
-        q = self.c_articles.find(db_filter,{"html":0,"text":0}).limit(limit)
+        #print(db_filter)
+        q = self.c_articles.find(db_filter,{"html":0,"text":0}).limit(limit).sort("scrap_date")
         count = q.count()
         return { 'articles': [ self.jsonable(p) for p in q ],
                  'count' : count }

news.py

@@ -24,24 +24,33 @@ class News():
     MATCHING_KW=[]
     TITLE=""
 
-    def __init__(self,url,sourceid, sourcename,kw,lang="en"):
+    def __init__(self,url,sourceid, sourcename,kw,lang="en",html=None):
         url=sanitizeUrl(url)
         self.URL=url
         self.SOURCENAME=sourcename
         self.SOURCE=sourceid
         self.MATCHING_KW=[]
 
-        a = Article(url, language=lang,keep_article_html=True,request_timeout=10)
+        a=None
         try:
-            a.download()
+            if html is None:
+                a = Article(url, language=lang,keep_article_html=True,request_timeout=10)
+                a.download()
+            else:
+                a= Article("")
+                a.set_html(html)
+
             a.parse()
             a.nlp()
         except Exception as e:
             print("ERROR parsing/downloading/nlp article")
             print(e)
+            if html is not None:
+                print(len(html))
             self.ERROR=True
             self.MATCHES=False
             return
+
         a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
         
         self.SCRAP_DATE=datetime.datetime.now()

paid_parser.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+import paid_sources.financial_times
+
+class PaidParser:
+    html=""
+    def __init__(self,url):
+        if "ft.com" in url:
+            self.html=paid_sources.financial_times.parse_ft(url)

paid_sources/financial_times.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+import requests
+
+s_r = requests.session()
+login_url = "https://accounts.ft.com/login"
+payload = {
+        "email" : "rhwang@marketplatforms.com",
+        "password": "PymntsFT2015", 
+        "Sign+in":"",
+        "rememberMe": "false"
+        }
+
+def parse_ft(link):
+    result = s_r.post(login_url, data=payload, headers = dict(referer=login_url))
+    result = s_r.get(link)
+    return result.content.decode("utf-8")

parser.py

@@ -6,8 +6,11 @@ from db import db
 from source import Source
 from linklist import LinkList
 from news import News
+from paid_parser import PaidParser
 
 def parseSource(s):
+    global q
+    q=Queue()
     num_worker_threads=10
     sel=None
     rss=False
@@ -19,49 +22,58 @@ def parseSource(s):
 
     kw=d.get_keywords(s["category"])
     if l is not None:
-        print("[Source] %s: %d total articles" % ( s["name"], len(l.links)))
         for a in l.links:
             if not d.article_exists(a):
                 q.put(a)
-        print("[Source] %s: %d new articles" % ( s["name"], q.qsize()))
+        print("[Source] %s: %d/%d new/total articles" % ( s["name"], q.qsize(), len(l.links)))
 
-
-    num_worker_threads=min(10,q.qsize())
     if not q.empty():
+        num_worker_threads=min(10,q.qsize())
         threads=[]
         for i in range(num_worker_threads):
-            t = threading.Thread(target=news_worker,args=[s["_id"],s["name"],kw])
+            t = threading.Thread(target=news_worker,args=[s,kw])
             t.daemon=True
             t.start()
             threads.append(t)
     
-        print("Waiting for %d queue elements" % q.qsize())
+#        print("Waiting for %d queue elements" % q.qsize())
         q.join() #block until all tasks are done
-        print("Finished waiting for the queue")
+#        print("Finished waiting for the queue")
 
         for i in range(num_worker_threads):
             q.put(None)
-        print("Waiting for %d threads" % len(threads))
         for t in threads:
             t.join(10)
-    print("[Source] %s => Finished parsing" %s["name"])
+#    print("[Source] %s => Finished parsing" %s["name"])
+
+def news_worker(source,kw):
+    source_id=source["_id"]
+    source_name=source["name"]
+    paid=False
+    if "paid" in source and source["paid"]:
+        paid=True
 
-def news_worker(source_id,source_name,kw):
     tname=threading.current_thread().name
     while True:
         try:
             item = q.get(timeout=5)
         except:
-            print("[%s] Empty queue" % tname)
+            print("Failed to get item")
+            q.task_done()
             break
+
         if item is None:
-            print("[%s] Empty queue (none)" % tname)
             q.task_done()
             break
 
         print("[Thread %s] %s" % (tname,item))
         try:
-            n=News(item,source_id,source_name,kw)
+            html=None
+            if paid:
+                p=PaidParser(item)
+                html=p.html
+
+            n=News(item,source_id,source_name,kw,html=html)
             print("[Thread %s] Finished" % tname)
             d.insert_article(n.get())
         except:

web.py

@@ -90,4 +90,4 @@ def create_task():
     return jsonify({'task': task}), 201
 
 if __name__ == '__main__':
-    app.run(debug=True)
+    app.run(host="0.0.0.0",debug=True)