alexis2/parser.py

#!/usr/bin/python3 
import sys
import re
import newspaper
import pymongo
import htmlmin
import feedparser
import datetime
import threading

from queue import Queue 
from frontpage import FrontPage
from newspaper import Article
from db import db
from source import Source

class LinkList():
    def __init__(self,url,selector=None,rss=False):
        self.links=[]
        if rss:
            feed = feedparser.parse(url)
            if feed["bozo"]:
                print("RSS ERROR. PANIC")
            #print(feed["bozo"]) FIXME: If bozo==1 => error
            self.links=[i["link"] for i in feed["items"]]
            print(self.links)
        else:
            if selector is None:
                #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida
                s = newspaper.Source(url,memoize_articles=False)
                s.download()
                s.parse()
                s.set_categories()
                s.download_categories()
                s.parse_categories()
                s.generate_articles()
    
                self.links=[a.url for a in s.articles] 
            else:
                f=FrontPage(url,selector)
                self.links=f.links
        self.links=self.purgeLinks(self.links)
        self.links=list(set(self.links)) #avoid dupes

    def purgeLinks(self,l):
        return [ sanitizeUrl(link) for link in l if not "presslist" in link.lower() and not "videolist" in link.lower() ]

class News():
    r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
    r_noise=re.compile(r"posted|Product|Google",flags=re.IGNORECASE)
    
    URL=""
    AUTHORS=[]
    SCRAP_DATE=None 
    PUBLISH_DATE=None
    SUMMARY=""
    TEXT=""
    HTML=""
    ERROR=False
    SOURCE=""
    WEIGHT_SUM=0
    MATCHES=False
    MATCHING_KW=[]

    def __init__(self,url,source,kw,lang="en"):
        url=sanitizeUrl(url)
        self.URL=url
        self.SOURCE=source

        a = Article(url, language=lang,keep_article_html=True)
        try:
            a.download()
            a.parse()
        except newspaper.article.ArticleException:
            print("ERROR parsing article")
            ERROR=True
            return
        except Exception:
            print("ERROR downloading article")
            ERROR=True
            return
        a.nlp()
        
        a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
        
        self.SCRAP_DATE=datetime.datetime.now()
        self.AUTHORS=a.authors
        self.PUBLISH_DATE=a.publish_date
        self.SUMMARY=a.summary
        self.TEXT=a.text
        self.HTML=htmlmin.minify(a.article_html,remove_empty_space=True)
        for k in kw:
            if re.match(k["pattern"], self.TEXT):
                self.WEIGHT_SUM+=k["weight"]
                self.MATCHING_KW.append(k["kw"])
                self.MATCHES=True
            

    def get(self):
        return {
                "url": self.URL,
                "source": self.SOURCE,
                "authors": self.AUTHORS,
                "scrap_date": self.SCRAP_DATE,
                "publish_date": self.PUBLISH_DATE,
                "summary": self.SUMMARY,
                "text": self.TEXT,
                "matches": self.MATCHES,
                "weight": self.WEIGHT_SUM,
                "kw": self.MATCHING_KW,
                "html": self.HTML 
                }

    def dedup(self,val):
        return list(set(val))

    def isNoise(self,v):
        if self.r_noise.match(v):
            return True
        return False
    def isComment(self,v):
        if self.r.match(v):
            return True
        return False
    
    def fix_author(self,a):
        return a.replace("_", " ").lower()


unsharer=re.compile(r"(\?share=|#).+$",flags=re.IGNORECASE)
def sanitizeUrl(url):
    global unsharer
    ret = re.sub(unsharer, "",url)
    return ret

def sanitizeSelector(s):
    s=s.replace(">a","> a")
    if re.match(r'=\w+\]',s) is None:
        return s

    ret=re.sub(r"=(\w+)]", r'="\1"]', s)
    return ret


def parseSource(s):
    sel=None
    rss=False
    if "selector" in s:
        sel=s["selector"]
    if "rss" in s:
        rss=bool(["rss"])
    l=LinkList(s["link"],sel,rss=rss)

    kw=d.get_keywords(s["category"])
    if l is not None:
        print("[Source] %s: %d total articles" % ( s["name"], len(l.links)))
        for a in l.links:
            if not d.article_exists(a):
                q.put(a)
        print("[Source] %s: %d new articles" % ( s["name"], q.qsize()))


    if not q.empty():
        threads=[]
        for i in range(num_worker_threads):
            t = threading.Thread(target=news_worker,args=[s["_id"],kw])
            t.daemon=True
            t.start()
            threads.append(t)
    
        q.join()
        for i in range(num_worker_threads):
            q.put(None)
        for t in threads:
            t.join()
    print("[Source] %s => Finished parsing" %s["name"])

def news_worker(source_id,kw):
    while True:
        item = q.get()
        if item is None:
            break
        n=News(item,source_id,kw)
        print("[Thread %s] %s" % (threading.current_thread().name,n.URL))
        d.insert_article(n.get())
        q.task_done()

q=Queue()
d=db()
num_worker_threads=20
for s in d.sources():
    parseSource(s)