瀏覽代碼

add forbes as paid

David 10 年之前
父節點
當前提交
e393f5b640
共有 2 個文件被更改,包括 27 次插入4 次删除
  1. 22 0
      paid_sources/forbes.py
  2. 5 4
      paid_sources/test_paid_article.py

+ 22 - 0
paid_sources/forbes.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+import requests
+import json
+
+def parse(link):
+    s_r = requests.session()
+    print_link=link+"/print/"
+    cookie = {'toURL':link,'refURL':'','referrer':''}
+    result = s_r.get(print_link, cookies=cookie)
+    out = result.content.decode("utf-8")
+    out = out.split("\n")
+
+    ret = {'title':"", 'body': ""}
+    for l in out:
+        if "fbs_settings.content = {" in l:
+            substr=l.split("=",1)[1]
+            substr="".join(substr.rsplit(";",1))
+            try:
+                ret = json.loads(substr) 
+            except:
+                pass
+    return "<html><head><title>%s</title></head><body><article>%s</article></body></html>" %(ret["title"], ret["body"] )

+ 5 - 4
paid_sources/test_paid_article.py

@@ -3,7 +3,7 @@ import newspaper
 from newspaper import Article,Config
 import sys
 
-url = "http://feeds.wsjonline.com/~r/wsj/deals/feed/~3/lnaq0DpDaYI/"
+url="http://www.forbes.com/pictures/fjle45edkdm/no-1-detroit-michigan/"
 if url == "":
     print("pls url")
     sys.exit(1)
@@ -18,13 +18,14 @@ config.browser_user_agent="Mozilla/5.0 (X11; Linux x86_64; rv:47.0) Gecko/201001
 config.verbose=True
 
 
-import wsj
+import forbes as parser
 print(url)
-html=wsj.parse(url)
+html=parser.parse(url)
 a = Article(url=url,config=config)
 a.download(html=html)
 #print("dl")
 a.parse()
-print("parsed")
+print(a.title)
 print(a.text)
+#print(a.html)
 #