alexis2/parser.py

#!/usr/bin/python3 
import sys
import re
import newspaper
import pymongo
import htmlmin
import feedparser
from frontpage import FrontPage
from newspaper import Article
from pymongo import MongoClient

class db():
    def __init__(self):
        self.client = MongoClient('localhost', 27017)
        self.col=self.client.alexis["sources"]

    def sources(self):
        return [ p for p in self.col.find({"rss":0}) ]

    def rss_sources(self):
        return [ p for p in self.col.find({"rss":1}) ]

class Source():
    URL=""
    BRAND=""
    DESCRIPTION=""
    CATEGORY=0
    def __init__(self,url,category,lang="en"):
        s=newspaper.build(url,language=lang,memoize_articles=False) 
        self.BRAND=s.brand
        self.URL=url
        self.DESCRIPTION=s.description
        self.CATEGORY=category

    def obj(self):
        return {"url":self.URL, "brand": self.BRAND, "desc": self.DESCRIPTION, "category":self.CATEGORY}

class LinkList():
    def __init__(self,url,selector=None,rss=False):
        self.links=[]
        if rss:
            feed = feedparser.parse(url)
            #print(feed["bozo"]) FIXME: If bozo==1 => error
            self.links=[i["link"] for i in feed["items"]]
            return
        if selector is None:
            #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida
            s = newspaper.Source(url,memoize_articles=False)
            s.download()
            s.parse()
            s.set_categories()
            s.download_categories()
            s.parse_categories()
            s.generate_articles()

            self.links=[a.url for a in s.articles]
        else:
            f=FrontPage(url,selector)
            self.links=f.links

class News():
    r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
    r_noise=re.compile(r"posted|Product|Google",flags=re.IGNORECASE)
    def __init__(self,url,lang="en"):
        url=self.sanitizeUrl(url)
        a = Article(url, language=lang,keep_article_html=True)
        a.download()
        a.parse()
        a.nlp()
        
        a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
        
        print(a.authors)
        print(a.publish_date)
        print(a.summary)
        print(a.keywords)
        #print(a.text)
        print(a.top_image)
        print(a.movies)
        print("_#_#_#_#_#_#_")
        print(htmlmin.minify(a.article_html,remove_empty_space=True))

    def sanitizeUrl(self,url):
        return url.replace("?share=google-plus-1","")

    def dedup(self,val):
        return list(set(val))

    def isNoise(self,v):
        if self.r_noise.match(v):
            return True
        return False
    def isComment(self,v):
        if self.r.match(v):
            return True
        return False
    
    def fix_author(self,a):
        return a.replace("_", " ").lower()


l=None
#d=db()
#l=LinkList("http://9to5google.com")
#l=LinkList("http://www.alibabagroup.com/en/news/press")
#l=LinkList("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-news", selector=".ccbnTblLnk")
#l=LinkList("http://about.americanexpress.com/news/pr/releases.aspx")
#NOT WORKING l=LinkList("http://www.americanbanker.com/")
#l=LinkList("http://officialandroid.blogspot.com/atom.xml",rss=True)
#l=LinkList("https://www.apple.com/pr/feeds/pr.rss",rss=True)
#l=LinkList("http://appleinsider.com/rss/news/",rss=True)
#l=LinkList("http://newsroom.bankofamerica.com/feeds/press_release/all/rss.xml",rss=True)
#l=LinkList("https://newsroom.fb.com/feed/",rss=True)
#n=News("http://appleinsider.com/articles/16/05/11/apple-again-rumored-to-turn-la-landmark-into-retail-store")
#n=News("http://9to5google.com/2016/05/11/nest-opensource-openthread-networking-protocol/?share=google-plus-1")
#n=News("http://www.alibabagroup.com/en/news/article?news=p150908b")
#n=News("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-newsArticle&ID=1250170")
#n=News("http://about.americanexpress.com/news/pr/2016/amex-dividend-payable-august-10-2016.aspx")
#n=News("http://www.apple.com/pr/library/2015/06/08Apple-Pay-Giving-Shoppers-Even-More-Ways-to-Pay.html")
#n=News("http://newsroom.bankofamerica.com/press-releases/small-business-banking/bank-america-survey-finds-small-business-owners-confidence-dow")
#n=News("http://newsroom.fb.com/news/2016/04/news-feed-fyi-more-articles-you-want-to-spend-time-viewing/")

#l=LinkList("https://www.blogger.com/feeds/10861780/posts/default",rss=True)
n=News("http://googleblog.blogspot.com/2016/05/translate-where-you-need-it-in-any-app.html")
if l is not None:
    for a in l.links:
        print(a)
    print(len(l.links))

sys.exit(1)
def sanitizeSelector(s):
    s=s.replace(">a","> a")
    if re.match(r'=\w+\]',s) is None:
        return s

    print(s)
    ret=re.sub(r"=(\w+)]", r'="\1"]', s)
    print(ret)
    return ret

for s in d.sources():
    s["title"]=sanitizeSelector(s["title"])

    l=LinkList(s["link"],s["title"])
    if len(l.links)==0:
        print(s["link"])
        print(s["title"])
        print("###################################")


#n=News("http://www.bloomberg.com/news/articles/2016-03-04/the-mystery-madoff-victims-who-left-2-5-billion-on-the-table")