alexis2/news.py

import re
import htmlmin
import datetime
import html

from functions import sanitizeUrl
from newspaper import Article,Config

class News():
    r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
    r_noise=re.compile(r"posted|Product|Google",flags=re.IGNORECASE)
    
    URL=""
    LOWERURL=""
    AUTHORS=[]
    SCRAP_DATE=None 
    PUBLISH_DATE=None
    SUMMARY=""
    TEXT=""
    HTML=""
    ERROR=False
    SOURCE=""
    SOURCENAME=""
    WEIGHT_SUM=0
    MATCHES=False
    MATCHING_KW=[]
    TITLE=""
    CATEGORY=0

    def __init__(self,url,sourceid, sourcename,sourcecategory,kw,lang="en",html=None):
        url=sanitizeUrl(url)
        self.URL=url
        self.LOWERURL=url.lower()
        self.SOURCENAME=sourcename
        self.SOURCE=sourceid
        self.CATEGORY=sourcecategory
        self.MATCHING_KW=[]

        config = Config()
        config.language=lang
        config.memoize_articles=False
        config.keep_article_html=True
        config.request_timeout=15
        config.fetch_images=False
        config.browser_user_agent="Mozilla/5.0 (X11; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"

        a=None
        try:
            a = Article(url=url, config=config)
            a.download(html=html)

            a.parse()
            a.nlp()
        except Exception as e:
            print("ERROR parsing/downloading/nlp article")
            print(e)
            print(url)
            print(sourcename)
            if html is not None:
                print(len(html))
            self.ERROR=True
            self.MATCHES=False
            return

        a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
        
        self.SCRAP_DATE=datetime.datetime.now()
        self.AUTHORS=a.authors
        self.PUBLISH_DATE=a.publish_date
        self.SUMMARY=a.summary
        self.TEXT=a.text
        self.TITLE=self.fix_title(a.title)
        self.HTML=htmlmin.minify(a.article_html,remove_empty_space=True)
#        if html is None:
#            self.HTML=htmlmin.minify(a.article_html,remove_empty_space=True)
#        else:
#            self.HTML=htmlmin.minify(html,remove_empty_space=True)
        self.ERROR=False
        lowertext=self.TEXT.lower()
        ret=match_keywords(kw,lowertext)
        self.MATCHES=ret["matches"]
        self.WEIGHT_SUM=ret["weight"]
        self.MATCHING_KW=ret["kw"]

    def get(self):
        return {
                "url": self.URL,
                "lowerurl": self.LOWERURL,
                "source": self.SOURCE,
                "source_name": self.SOURCENAME,
                "authors": self.AUTHORS,
                "scrap_date": self.SCRAP_DATE,
                "publish_date": self.PUBLISH_DATE,
                "summary": self.SUMMARY,
                "text": self.TEXT,
                "matches": self.MATCHES,
                "weight": self.WEIGHT_SUM,
                "kw": self.MATCHING_KW,
                "html": self.HTML, 
                "error": self.ERROR,
                "deleted": False,
                "category": self.CATEGORY,
                "title": self.TITLE
                }

    def fix_title(self,n):
        while(html.unescape(n)!=n):
            n=html.unescape(n)
        n=n.replace("&Dquot",'"')
        return n

    def dedup(self,val):
        return list(set(val))

    def isNoise(self,v):
        if self.r_noise.match(v):
            return True
        return False
    def isComment(self,v):
        if self.r.match(v):
            return True
        return False
    
    def fix_author(self,a):
        return a.replace("_", " ").lower()


def match_keywords(keywords, lowertext):
    w_sum=0
    matching=[]
    matches=False
    for k in keywords:
        num_matches=len( [ 1 for kwp in k["pattern"] if kwp.search(lowertext) ] )
        if num_matches==len(k["pattern"]):
            w_sum+=k["weight"]
            matching.append(k["kw"])
            matches=True

    ret=({"matches":matches,"weight":w_sum,"kw":matching})
    return ret