alexis2/parser.py

#!/usr/bin/python3 
import sys
import re
import newspaper
import pymongo
import htmlmin
import feedparser
from frontpage import FrontPage
from newspaper import Article
from pymongo import MongoClient

class db():
    def __init__(self):
        self.client = MongoClient('localhost', 27017)
        self.col=self.client.alexis["sources"]

    def sources(self):
        return [ p for p in self.col.find({"rss":0}) ]

    def rss_sources(self):
        return [ p for p in self.col.find({"rss":1}) ]

class Source():
    URL=""
    BRAND=""
    DESCRIPTION=""
    CATEGORY=0
    def __init__(self,url,category,lang="en"):
        s=newspaper.build(url,language=lang,memoize_articles=False) 
        self.BRAND=s.brand
        self.URL=url
        self.DESCRIPTION=s.description
        self.CATEGORY=category

    def obj(self):
        return {"url":self.URL, "brand": self.BRAND, "desc": self.DESCRIPTION, "category":self.CATEGORY}

class LinkList():
    def __init__(self,url,selector=None,rss=False):
        if rss:
            feed = feedparser.parse(url)
            print(feed["bozo"])
            #print(feed["channel"])
            for i in feed["items"]:
                print(i["date"])
                print(i["date_parsed"])
                print(i["title"])
                print(i["summary"])
                print(i["link"])
            self.links=[]
            return
        if selector is None:
            #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida
            s = newspaper.Source(url,memoize_articles=False)
            s.download()
            s.parse()
            s.set_categories()
            s.download_categories()
            s.parse_categories()
            s.generate_articles()

            for a in s.articles:
                self.articlePrinter(a)
            print(s.size())
        else:
            f=FrontPage(url,selector)
            for l in f.links:
                print(l)
            self.links=f.links

    def articlePrinter(self,a):
        print(a.url)
#    def getList(self):
#        ret=[]
#        for art in self.source.articles:
#            print(art)
#            print(art.url)
#            ret.append(art.url)
#        return ret


class News():
    r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
    def __init__(self,url,lang="en"):
        url=self.sanitizeUrl(url)
        a = Article(url, language=lang,keep_article_html=True)
        a.download()
        a.parse()
        a.nlp()
        
        a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value)])
        
        print(a.authors)
        print(a.publish_date)
        print(a.summary)
        print(a.keywords)
        #print(a.text)
        print(a.top_image)
        print(a.movies)
        print("_#_#_#_#_#_#_")
        print(htmlmin.minify(a.article_html,remove_empty_space=True))

    def sanitizeUrl(self,url):
        return url.replace("?share=google-plus-1","")

    def dedup(self,val):
        return list(set(val))

    def isComment(self,v):
        if self.r.match(v):
            return True
        return False
    
    def fix_author(self,a):
        return a.replace("_", " ").lower()


#d=db()
#l=LinkList("http://9to5google.com")
#l=LinkList("http://www.alibabagroup.com/en/news/press")
#l=LinkList("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-news", selector=".ccbnTblLnk")
#l=LinkList("http://about.americanexpress.com/news/pr/releases.aspx")
#NOT WORKING l=LinkList("http://www.americanbanker.com/")
l=LinkList("http://officialandroid.blogspot.com/atom.xml",rss=True)
#n=News("http://9to5google.com/2016/05/11/nest-opensource-openthread-networking-protocol/?share=google-plus-1")
#n=News("http://www.alibabagroup.com/en/news/article?news=p150908b")
#n=News("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-newsArticle&ID=1250170")
#n=News("http://about.americanexpress.com/news/pr/2016/amex-dividend-payable-august-10-2016.aspx")
sys.exit(1)
def sanitizeSelector(s):
    s=s.replace(">a","> a")
    if re.match(r'=\w+\]',s) is None:
        return s

    print(s)
    ret=re.sub(r"=(\w+)]", r'="\1"]', s)
    print(ret)
    return ret

for s in d.sources():
    s["title"]=sanitizeSelector(s["title"])

    l=LinkList(s["link"],s["title"])
    if len(l.links)==0:
        print(s["link"])
        print(s["title"])
        print("###################################")


#n=News("http://www.bloomberg.com/news/articles/2016-03-04/the-mystery-madoff-victims-who-left-2-5-billion-on-the-table")