alexis2/parser.py

#!/usr/bin/python3 
import sys
import re
import newspaper
import pymongo
import htmlmin
import feedparser
from frontpage import FrontPage
from newspaper import Article
from db import db
from source import Source

class LinkList():
    def __init__(self,url,selector=None,rss=False):
        self.links=[]
        if rss:
            feed = feedparser.parse(url)
            #print(feed["bozo"]) FIXME: If bozo==1 => error
            self.links=[i["link"] for i in feed["items"]]
        else:
            if selector is None:
                #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida
                s = newspaper.Source(url,memoize_articles=False)
                s.download()
                s.parse()
                s.set_categories()
                s.download_categories()
                s.parse_categories()
                s.generate_articles()
    
                self.links=list(set([a.url for a in s.articles])) #avoid dupes
            else:
                f=FrontPage(url,selector)
                self.links=f.links

        self.links=self.purgeLinks(self.links)

    def purgeLinks(self,l):
        return [ sanitizeUrl(link) for link in l if not "presslist" in link.lower() and not "videolist" in link.lower() ]

class News():
    r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
    r_noise=re.compile(r"posted|Product|Google",flags=re.IGNORECASE)
    def __init__(self,url,lang="en"):
        url=sanitizeUrl(url)
        a = Article(url, language=lang,keep_article_html=True)
        a.download()
        a.parse()
        a.nlp()
        
        a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
        
        print(a.authors)
        print(a.publish_date)
        print(a.summary)
        print(a.keywords)
        #print(a.text) #FIXME
        print(a.top_image)
        print(a.movies)
        print(htmlmin.minify(a.article_html,remove_empty_space=True))


    def dedup(self,val):
        return list(set(val))

    def isNoise(self,v):
        if self.r_noise.match(v):
            return True
        return False
    def isComment(self,v):
        if self.r.match(v):
            return True
        return False
    
    def fix_author(self,a):
        return a.replace("_", " ").lower()


l=None
#d=db()
#l=LinkList("http://9to5google.com")
#l=LinkList("http://www.alibabagroup.com/en/news/press")
#l=LinkList("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-news", selector=".ccbnTblLnk")
#l=LinkList("http://about.americanexpress.com/news/pr/releases.aspx")
#NOT WORKING l=LinkList("http://www.americanbanker.com/")
#l=LinkList("http://officialandroid.blogspot.com/atom.xml",rss=True)
#l=LinkList("https://www.apple.com/pr/feeds/pr.rss",rss=True)
#l=LinkList("http://appleinsider.com/rss/news/",rss=True)
#l=LinkList("http://newsroom.bankofamerica.com/feeds/press_release/all/rss.xml",rss=True)
#l=LinkList("https://newsroom.fb.com/feed/",rss=True)
#n=News("http://appleinsider.com/articles/16/05/11/apple-again-rumored-to-turn-la-landmark-into-retail-store")
#n=News("http://9to5google.com/2016/05/11/nest-opensource-openthread-networking-protocol/?share=google-plus-1")
#n=News("http://www.alibabagroup.com/en/news/article?news=p150908b")
#n=News("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-newsArticle&ID=1250170")
#n=News("http://about.americanexpress.com/news/pr/2016/amex-dividend-payable-august-10-2016.aspx")
#n=News("http://www.apple.com/pr/library/2015/06/08Apple-Pay-Giving-Shoppers-Even-More-Ways-to-Pay.html")
#n=News("http://newsroom.bankofamerica.com/press-releases/small-business-banking/bank-america-survey-finds-small-business-owners-confidence-dow")
#n=News("http://newsroom.fb.com/news/2016/04/news-feed-fyi-more-articles-you-want-to-spend-time-viewing/")

#l=LinkList("https://www.blogger.com/feeds/10861780/posts/default",rss=True)
#n=News("http://googleblog.blogspot.com/2016/05/translate-where-you-need-it-in-any-app.html") #FIXME

#l=LinkList("https://apps.shareholder.com/rss/rss.aspx?channels=14&companyid=ONE",rss=True)
#n=News("http://investor.shareholder.com/jpmorganchase/press/releasedetail.cfm?ReleaseID=970074")

#l=LinkList("http://www.samsungmobilepress.com/press/pressList.asp")
#n=News("http://www.samsungmobilepress.com/press/Samsungs-Connected-Ecosystem-Accelerates-the-Growth-of-the-Gear-S2-Applications?2016-03-22")

#l=LinkList("https://news.starbucks.com/feeds/news")
#l=LinkList("http://www.techinsider.io/rss",rss=True)

#l=LinkList("https://newsroom.uber.com/feed/", rss=True)

#if l is not None:
#    for a in l.links:
#        print(a)
#    print(len(l.links))
#    n=News(l.links[0])
#
#sys.exit(1)

unsharer=re.compile(r"\?share=.+$",flags=re.IGNORECASE)
def sanitizeUrl(url):
    global unsharer
    return re.sub(unsharer, "",url)
    #return url.replace("?share=google-plus-1","").replace("?share=facebook","")

def sanitizeSelector(s):
    s=s.replace(">a","> a")
    if re.match(r'=\w+\]',s) is None:
        return s

    print(s)
    ret=re.sub(r"=(\w+)]", r'="\1"]', s)
    print(ret)
    return ret

d=db()
for s in d.sources():
    print(s["name"])
    sel=None
    rss=False
    if "selector" in s:
        sel=s["selector"]
    if "rss" in s:
        rss=bool(["rss"])
    l=LinkList(s["link"],sel,rss=rss)

    if l is not None:
        for a in l.links:
            print(a)
        print(len(l.links))
        n=News(l.links[0])
    break