purge tests + news get()

parser.py

@@ -17,6 +17,7 @@ class LinkList():
             feed = feedparser.parse(url)
             #print(feed["bozo"]) FIXME: If bozo==1 => error
             self.links=[i["link"] for i in feed["items"]]
+            print(self.links)
         else:
             if selector is None:
                 #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida
@@ -41,8 +42,20 @@ class LinkList():
 class News():
     r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
     r_noise=re.compile(r"posted|Product|Google",flags=re.IGNORECASE)
+    
+    URL=""
+    AUTHORS=[]
+    SCRAP_DATE=None 
+    PUBLISH_DATE=None
+    SUMMARY=""
+    TEXT=""
+    HTML=""
+    ERROR=False
+
     def __init__(self,url,lang="en"):
         url=sanitizeUrl(url)
+        self.URL=url
+
         a = Article(url, language=lang,keep_article_html=True)
         a.download()
         a.parse()
@@ -50,15 +63,21 @@ class News():
         
         a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
         
-        print(a.authors)
-        print(a.publish_date)
-        print(a.summary)
-        print(a.keywords)
-        #print(a.text) #FIXME
-        print(a.top_image)
-        print(a.movies)
-        print(htmlmin.minify(a.article_html,remove_empty_space=True))
-
+        self.AUTHORS=a.authors
+        self.PUBLISH_DATE=a.publish_date
+        self.SUMMARY=a.summary
+        self.TEXT=a.text
+        self.HTML=htmlmin.minify(a.article_html,remove_empty_space=True)
+
+    def get(self):
+        return {
+                "authors": self.AUTHORS,
+                "scrap_date": self.SCRAP_DATE,
+                "publish_date": self.PUBLISH_DATE,
+                "summary": self.SUMMARY,
+                "text": self.TEXT,
+                "html": self.HTML 
+                }
 
     def dedup(self,val):
         return list(set(val))
@@ -76,63 +95,17 @@ class News():
         return a.replace("_", " ").lower()
 
 
-l=None
-#d=db()
-#l=LinkList("http://9to5google.com")
-#l=LinkList("http://www.alibabagroup.com/en/news/press")
-#l=LinkList("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-news", selector=".ccbnTblLnk")
-#l=LinkList("http://about.americanexpress.com/news/pr/releases.aspx")
-#NOT WORKING l=LinkList("http://www.americanbanker.com/")
-#l=LinkList("http://officialandroid.blogspot.com/atom.xml",rss=True)
-#l=LinkList("https://www.apple.com/pr/feeds/pr.rss",rss=True)
-#l=LinkList("http://appleinsider.com/rss/news/",rss=True)
-#l=LinkList("http://newsroom.bankofamerica.com/feeds/press_release/all/rss.xml",rss=True)
-#l=LinkList("https://newsroom.fb.com/feed/",rss=True)
-#n=News("http://appleinsider.com/articles/16/05/11/apple-again-rumored-to-turn-la-landmark-into-retail-store")
-#n=News("http://9to5google.com/2016/05/11/nest-opensource-openthread-networking-protocol/?share=google-plus-1")
-#n=News("http://www.alibabagroup.com/en/news/article?news=p150908b")
-#n=News("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-newsArticle&ID=1250170")
-#n=News("http://about.americanexpress.com/news/pr/2016/amex-dividend-payable-august-10-2016.aspx")
-#n=News("http://www.apple.com/pr/library/2015/06/08Apple-Pay-Giving-Shoppers-Even-More-Ways-to-Pay.html")
-#n=News("http://newsroom.bankofamerica.com/press-releases/small-business-banking/bank-america-survey-finds-small-business-owners-confidence-dow")
-#n=News("http://newsroom.fb.com/news/2016/04/news-feed-fyi-more-articles-you-want-to-spend-time-viewing/")
-
-#l=LinkList("https://www.blogger.com/feeds/10861780/posts/default",rss=True)
-#n=News("http://googleblog.blogspot.com/2016/05/translate-where-you-need-it-in-any-app.html") #FIXME
-
-#l=LinkList("https://apps.shareholder.com/rss/rss.aspx?channels=14&companyid=ONE",rss=True)
-#n=News("http://investor.shareholder.com/jpmorganchase/press/releasedetail.cfm?ReleaseID=970074")
-
-#l=LinkList("http://www.samsungmobilepress.com/press/pressList.asp")
-#n=News("http://www.samsungmobilepress.com/press/Samsungs-Connected-Ecosystem-Accelerates-the-Growth-of-the-Gear-S2-Applications?2016-03-22")
-
-#l=LinkList("https://news.starbucks.com/feeds/news")
-#l=LinkList("http://www.techinsider.io/rss",rss=True)
-
-#l=LinkList("https://newsroom.uber.com/feed/", rss=True)
-
-#if l is not None:
-#    for a in l.links:
-#        print(a)
-#    print(len(l.links))
-#    n=News(l.links[0])
-#
-#sys.exit(1)
-
 unsharer=re.compile(r"\?share=.+$",flags=re.IGNORECASE)
 def sanitizeUrl(url):
     global unsharer
     return re.sub(unsharer, "",url)
-    #return url.replace("?share=google-plus-1","").replace("?share=facebook","")
 
 def sanitizeSelector(s):
     s=s.replace(">a","> a")
     if re.match(r'=\w+\]',s) is None:
         return s
 
-    print(s)
     ret=re.sub(r"=(\w+)]", r'="\1"]', s)
-    print(ret)
     return ret
 
 d=db()
@@ -151,4 +124,3 @@ for s in d.sources():
             print(a)
         print(len(l.links))
         n=News(l.links[0])
-    break