add basic paidParser support

paid_parser.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+import paid_sources.financial_times
+
+class PaidParser:
+    html=""
+    def __init__(self,url):
+        if "ft.com" in url:
+            self.html=paid_sources.financial_times.parse_ft(url)

paid_sources/financial_times.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+import requests
+
+s_r = requests.session()
+login_url = "https://accounts.ft.com/login"
+payload = {
+        "email" : "rhwang@marketplatforms.com",
+        "password": "PymntsFT2015", 
+        "Sign+in":"",
+        "rememberMe": "false"
+        }
+
+def parse_ft(link):
+    result = s_r.post(login_url, data=payload, headers = dict(referer=login_url))
+    result = s_r.get(link)
+    return result.content.decode("utf-8")

parser.py

@@ -6,6 +6,7 @@ from db import db
 from source import Source
 from linklist import LinkList
 from news import News
+from paid_parser import PaidParser
 
 def parseSource(s):
     num_worker_threads=10
@@ -19,49 +20,55 @@ def parseSource(s):
 
     kw=d.get_keywords(s["category"])
     if l is not None:
-        print("[Source] %s: %d total articles" % ( s["name"], len(l.links)))
         for a in l.links:
             if not d.article_exists(a):
                 q.put(a)
-        print("[Source] %s: %d new articles" % ( s["name"], q.qsize()))
-
+        print("[Source] %s: %d/%d new/total articles" % ( s["name"], q.qsize(), len(l.links)))
 
     num_worker_threads=min(10,q.qsize())
     if not q.empty():
         threads=[]
         for i in range(num_worker_threads):
-            t = threading.Thread(target=news_worker,args=[s["_id"],s["name"],kw])
+            t = threading.Thread(target=news_worker,args=[s,kw])
             t.daemon=True
             t.start()
             threads.append(t)
     
-        print("Waiting for %d queue elements" % q.qsize())
+#        print("Waiting for %d queue elements" % q.qsize())
         q.join() #block until all tasks are done
-        print("Finished waiting for the queue")
+#        print("Finished waiting for the queue")
 
         for i in range(num_worker_threads):
             q.put(None)
-        print("Waiting for %d threads" % len(threads))
         for t in threads:
             t.join(10)
-    print("[Source] %s => Finished parsing" %s["name"])
+#    print("[Source] %s => Finished parsing" %s["name"])
+
+def news_worker(source,kw):
+    source_id=source["_id"]
+    source_name=source["name"]
+    paid=False
+    if "paid" in source and source["paid"]:
+        paid=True
 
-def news_worker(source_id,source_name,kw):
     tname=threading.current_thread().name
     while True:
         try:
             item = q.get(timeout=5)
         except:
-            print("[%s] Empty queue" % tname)
             break
         if item is None:
-            print("[%s] Empty queue (none)" % tname)
             q.task_done()
             break
 
         print("[Thread %s] %s" % (tname,item))
         try:
-            n=News(item,source_id,source_name,kw)
+            html=None
+            if paid:
+                p=PaidParser(item)
+                html=p.html
+
+            n=News(item,source_id,source_name,kw,html=html)
             print("[Thread %s] Finished" % tname)
             d.insert_article(n.get())
         except: