alexis2/parser.py

#!/usr/bin/python3 
import sys
import re
import newspaper
import pymongo
import htmlmin
import feedparser
from frontpage import FrontPage
from newspaper import Article
from db import db
from source import Source

class LinkList():
    def __init__(self,url,selector=None,rss=False):
        self.links=[]
        if rss:
            feed = feedparser.parse(url)
            #print(feed["bozo"]) FIXME: If bozo==1 => error
            self.links=[i["link"] for i in feed["items"]]
        else:
            if selector is None:
                #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida
                s = newspaper.Source(url,memoize_articles=False)
                s.download()
                s.parse()
                s.set_categories()
                s.download_categories()
                s.parse_categories()
                s.generate_articles()
    
                self.links=[a.url for a in s.articles]
            else:
                f=FrontPage(url,selector)
                self.links=f.links

        self.links=self.purgeLinks(self.links)

    def purgeLinks(self,l):
        return [ link for link in l if not "presslist" in link.lower() and not "videolist" in link.lower() ]

class News():
    r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
    r_noise=re.compile(r"posted|Product|Google",flags=re.IGNORECASE)
    def __init__(self,url,lang="en"):
        url=self.sanitizeUrl(url)
        a = Article(url, language=lang,keep_article_html=True)
        a.download()
        a.parse()
        a.nlp()
        
        a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
        
        print(a.authors)
        print(a.publish_date)
        print(a.summary)
        print(a.keywords)
        #print(a.text) #FIXME
        print(a.top_image)
        print(a.movies)
        print(htmlmin.minify(a.article_html,remove_empty_space=True))

    def sanitizeUrl(self,url):
        return url.replace("?share=google-plus-1","")

    def dedup(self,val):
        return list(set(val))

    def isNoise(self,v):
        if self.r_noise.match(v):
            return True
        return False
    def isComment(self,v):
        if self.r.match(v):
            return True
        return False
    
    def fix_author(self,a):
        return a.replace("_", " ").lower()


l=None
#d=db()
#l=LinkList("http://9to5google.com")
#l=LinkList("http://www.alibabagroup.com/en/news/press")
#l=LinkList("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-news", selector=".ccbnTblLnk")
#l=LinkList("http://about.americanexpress.com/news/pr/releases.aspx")
#NOT WORKING l=LinkList("http://www.americanbanker.com/")
#l=LinkList("http://officialandroid.blogspot.com/atom.xml",rss=True)
#l=LinkList("https://www.apple.com/pr/feeds/pr.rss",rss=True)
#l=LinkList("http://appleinsider.com/rss/news/",rss=True)
#l=LinkList("http://newsroom.bankofamerica.com/feeds/press_release/all/rss.xml",rss=True)
#l=LinkList("https://newsroom.fb.com/feed/",rss=True)
#n=News("http://appleinsider.com/articles/16/05/11/apple-again-rumored-to-turn-la-landmark-into-retail-store")
#n=News("http://9to5google.com/2016/05/11/nest-opensource-openthread-networking-protocol/?share=google-plus-1")
#n=News("http://www.alibabagroup.com/en/news/article?news=p150908b")
#n=News("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-newsArticle&ID=1250170")
#n=News("http://about.americanexpress.com/news/pr/2016/amex-dividend-payable-august-10-2016.aspx")
#n=News("http://www.apple.com/pr/library/2015/06/08Apple-Pay-Giving-Shoppers-Even-More-Ways-to-Pay.html")
#n=News("http://newsroom.bankofamerica.com/press-releases/small-business-banking/bank-america-survey-finds-small-business-owners-confidence-dow")
#n=News("http://newsroom.fb.com/news/2016/04/news-feed-fyi-more-articles-you-want-to-spend-time-viewing/")

#l=LinkList("https://www.blogger.com/feeds/10861780/posts/default",rss=True)
#n=News("http://googleblog.blogspot.com/2016/05/translate-where-you-need-it-in-any-app.html") #FIXME

#l=LinkList("https://apps.shareholder.com/rss/rss.aspx?channels=14&companyid=ONE",rss=True)
#n=News("http://investor.shareholder.com/jpmorganchase/press/releasedetail.cfm?ReleaseID=970074")

#l=LinkList("http://www.samsungmobilepress.com/press/pressList.asp")
#n=News("http://www.samsungmobilepress.com/press/Samsungs-Connected-Ecosystem-Accelerates-the-Growth-of-the-Gear-S2-Applications?2016-03-22")

#l=LinkList("https://news.starbucks.com/feeds/news")
#l=LinkList("http://www.techinsider.io/rss",rss=True)

l=LinkList("https://newsroom.uber.com/feed/", rss=True)

if l is not None:
    for a in l.links:
        print(a)
    print(len(l.links))
    n=News(l.links[0])

sys.exit(1)
def sanitizeSelector(s):
    s=s.replace(">a","> a")
    if re.match(r'=\w+\]',s) is None:
        return s

    print(s)
    ret=re.sub(r"=(\w+)]", r'="\1"]', s)
    print(ret)
    return ret

for s in d.sources():
    s["title"]=sanitizeSelector(s["title"])

    l=LinkList(s["link"],s["title"])
    if len(l.links)==0:
        print(s["link"])
        print(s["title"])
        print("###################################")


#n=News("http://www.bloomberg.com/news/articles/2016-03-04/the-mystery-madoff-victims-who-left-2-5-billion-on-the-table")