alexis2/parser.py

#!/usr/bin/python3 

import sys
import re
import newspaper
import pymongo
from frontpage import FrontPage
from newspaper import Article
from pymongo import MongoClient

class db():
    def __init__(self):
        self.client = MongoClient('localhost', 27017)
        self.col=self.client.alexis["sources"]

    def sources(self):
        return [ p for p in self.col.find({"rss":0}) ]

    def rss_sources(self):
        return [ p for p in self.col.find({"rss":1}) ]

class LinkList():
    def __init__(self,url,selector=None):
        if selector is None:
            s=newspaper.build(url)
            print(url)
            for a in s.articles:
                print(a)
        else:
            f=FrontPage(url,selector)
            self.links=f.links

#    def getList(self):
#        ret=[]
#        for art in self.source.articles:
#            print(art)
#            print(art.url)
#            ret.append(art.url)
#        return ret


class News():
    r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
    def __init__(self,url):
        a = Article(url, language='en')
        a.download()
        a.parse()
        a.nlp()
        
        a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value)])
        
        print(a.authors)
        print(a.publish_date)
        print(a.summary)
        print(a.keywords)
        #print(a.text)
        print(a.top_image)
        print(a.movies)


    def dedup(self,val):
        return list(set(val))

    def isComment(self,v):
        if self.r.match(v):
            return True
        return False
    
    def fix_author(self,a):
        return a.replace("_", " ").lower()


d=db()
def sanitizeSelector(s):
    s=s.replace(">a","> a")
    if re.match(r'=\w+\]',s) is None:
        return s

    print(s)
    ret=re.sub(r"=(\w+)]", r'="\1"]', s)
    print(ret)
    return ret

for s in d.sources():
    s["title"]=sanitizeSelector(s["title"])

    l=LinkList(s["link"],s["title"])
    if len(l.links)==0:
        print(s["link"])
        print(s["title"])
        print("###################################")


#n=News("http://www.bloomberg.com/news/articles/2016-03-04/the-mystery-madoff-victims-who-left-2-5-billion-on-the-table")