initial v2

frontpage.py

@@ -0,0 +1,38 @@
+#!/usr/bin/python3
+from bs4 import BeautifulSoup
+import requests
+
+
+class FrontPage():
+    def getPage(self,url):
+        r = requests.get(url)
+        return r.content
+    
+    def getDomain(self,url):
+        parts = url.split('//', 1)
+        return parts[0]+'//'+parts[1].split('/', 1)[0]
+    
+    def fixHref(self,url,baseurl):
+        if url.startswith("http"):
+            return url.strip()
+
+        if url.startswith("/"):
+            domain=self.getDomain(baseurl)
+            if domain.endswith("/") and url.startswith("/"):
+                url=url[1:]
+
+            url=domain+url
+
+        return url.strip()
+
+    def __init__(self,url,selector):
+        soup = BeautifulSoup(self.getPage(url), 'html.parser')
+        links=soup.select(selector)
+        self.links=[]
+        for l in links:
+            if not l.has_attr("href"):
+                print("PANIC")
+                print(l)
+                print(url)
+                continue
+            self.links.append({ "href": self.fixHref(l["href"],url),"title":l.get_text() })

parser.py

@@ -0,0 +1,94 @@
+#!/usr/bin/python3 
+
+import sys
+import re
+import newspaper
+import pymongo
+from frontpage import FrontPage
+from newspaper import Article
+from pymongo import MongoClient
+
+class db():
+    def __init__(self):
+        self.client = MongoClient('localhost', 27017)
+        self.col=self.client.alexis["sources"]
+
+    def sources(self):
+        return [ p for p in self.col.find({"rss":0}) ]
+
+    def rss_sources(self):
+        return [ p for p in self.col.find({"rss":1}) ]
+
+class LinkList():
+    def __init__(self,url,selector=None):
+        if selector is None:
+            s=newspaper.build(url)
+            print(url)
+            for a in s.articles:
+                print(a)
+        else:
+            f=FrontPage(url,selector)
+            self.links=f.links
+
+#    def getList(self):
+#        ret=[]
+#        for art in self.source.articles:
+#            print(art)
+#            print(art.url)
+#            ret.append(art.url)
+#        return ret
+
+
+class News():
+    r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
+    def __init__(self,url):
+        a = Article(url, language='en')
+        a.download()
+        a.parse()
+        a.nlp()
+        
+        a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value)])
+        
+        print(a.authors)
+        print(a.publish_date)
+        print(a.summary)
+        print(a.keywords)
+        #print(a.text)
+        print(a.top_image)
+        print(a.movies)
+
+
+    def dedup(self,val):
+        return list(set(val))
+
+    def isComment(self,v):
+        if self.r.match(v):
+            return True
+        return False
+    
+    def fix_author(self,a):
+        return a.replace("_", " ").lower()
+
+
+d=db()
+def sanitizeSelector(s):
+    s=s.replace(">a","> a")
+    if re.match(r'=\w+\]',s) is None:
+        return s
+
+    print(s)
+    ret=re.sub(r"=(\w+)]", r'="\1"]', s)
+    print(ret)
+    return ret
+
+for s in d.sources():
+    s["title"]=sanitizeSelector(s["title"])
+
+    l=LinkList(s["link"],s["title"])
+    if len(l.links)==0:
+        print(s["link"])
+        print(s["title"])
+        print("###################################")
+
+
+#n=News("http://www.bloomberg.com/news/articles/2016-03-04/the-mystery-madoff-victims-who-left-2-5-billion-on-the-table")