add source class, basic rss parsing, misc newspaper options

parser.py

@@ -1,9 +1,10 @@
 #!/usr/bin/python3 
-
 import sys
 import re
 import newspaper
 import pymongo
+import htmlmin
+import feedparser
 from frontpage import FrontPage
 from newspaper import Article
 from pymongo import MongoClient
@@ -19,17 +20,56 @@ class db():
     def rss_sources(self):
         return [ p for p in self.col.find({"rss":1}) ]
 
+class Source():
+    URL=""
+    BRAND=""
+    DESCRIPTION=""
+    CATEGORY=0
+    def __init__(self,url,category,lang="en"):
+        s=newspaper.build(url,language=lang,memoize_articles=False) 
+        self.BRAND=s.brand
+        self.URL=url
+        self.DESCRIPTION=s.description
+        self.CATEGORY=category
+
+    def obj(self):
+        return {"url":self.URL, "brand": self.BRAND, "desc": self.DESCRIPTION, "category":self.CATEGORY}
+
 class LinkList():
-    def __init__(self,url,selector=None):
+    def __init__(self,url,selector=None,rss=False):
+        if rss:
+            feed = feedparser.parse(url)
+            print(feed["bozo"])
+            #print(feed["channel"])
+            for i in feed["items"]:
+                print(i["date"])
+                print(i["date_parsed"])
+                print(i["title"])
+                print(i["summary"])
+                print(i["link"])
+            self.links=[]
+            return
         if selector is None:
-            s=newspaper.build(url)
-            print(url)
+            #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida
+            s = newspaper.Source(url,memoize_articles=False)
+            s.download()
+            s.parse()
+            s.set_categories()
+            s.download_categories()
+            s.parse_categories()
+            s.generate_articles()
+
             for a in s.articles:
-                print(a)
+                self.articlePrinter(a)
+            print(s.size())
         else:
             f=FrontPage(url,selector)
+            for l in f.links:
+                print(l)
             self.links=f.links
 
+    def articlePrinter(self,a):
+        print(a.url)
 #    def getList(self):
 #        ret=[]
 #        for art in self.source.articles:
@@ -41,8 +81,9 @@ class LinkList():
 
 class News():
     r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
-    def __init__(self,url):
-        a = Article(url, language='en')
+    def __init__(self,url,lang="en"):
+        url=self.sanitizeUrl(url)
+        a = Article(url, language=lang,keep_article_html=True)
         a.download()
         a.parse()
         a.nlp()
@@ -56,7 +97,11 @@ class News():
         #print(a.text)
         print(a.top_image)
         print(a.movies)
+        print("_#_#_#_#_#_#_")
+        print(htmlmin.minify(a.article_html,remove_empty_space=True))
 
+    def sanitizeUrl(self,url):
+        return url.replace("?share=google-plus-1","")
 
     def dedup(self,val):
         return list(set(val))
@@ -70,7 +115,18 @@ class News():
         return a.replace("_", " ").lower()
 
 
-d=db()
+#d=db()
+#l=LinkList("http://9to5google.com")
+#l=LinkList("http://www.alibabagroup.com/en/news/press")
+#l=LinkList("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-news", selector=".ccbnTblLnk")
+#l=LinkList("http://about.americanexpress.com/news/pr/releases.aspx")
+#NOT WORKING l=LinkList("http://www.americanbanker.com/")
+l=LinkList("http://officialandroid.blogspot.com/atom.xml",rss=True)
+#n=News("http://9to5google.com/2016/05/11/nest-opensource-openthread-networking-protocol/?share=google-plus-1")
+#n=News("http://www.alibabagroup.com/en/news/article?news=p150908b")
+#n=News("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-newsArticle&ID=1250170")
+#n=News("http://about.americanexpress.com/news/pr/2016/amex-dividend-payable-august-10-2016.aspx")
+sys.exit(1)
 def sanitizeSelector(s):
     s=s.replace(">a","> a")
     if re.match(r'=\w+\]',s) is None: