lint

db.py

@@ -1,113 +1,120 @@
-from pymongo import MongoClient,errors,DESCENDING,ASCENDING
+from pymongo import MongoClient, errors, DESCENDING, ASCENDING
 from bson.objectid import ObjectId
 import datetime
 import re
 import news
 import pytz
 
+
 class db():
-    DB="alexis"
+    DB = "alexis"
+
     def __init__(self):
         self.client = MongoClient('localhost', 27017)
-        self.c_sources=self.client[self.DB]["sources"]
-        self.c_articles=self.client[self.DB]["articles"]
+        self.c_sources = self.client[self.DB]["sources"]
+        self.c_articles = self.client[self.DB]["articles"]
 
-    def change_timezone(self,el):
-        gmt = pytz.timezone("America/Buenos_Aires") #FIXME
+    def change_timezone(self, el):
+        gmt = pytz.timezone("America/Buenos_Aires")  # FIXME
         est = pytz.timezone('US/Eastern')
         fmt = '%Y-%m-%d %H:%M:%S %Z%z'
 
-        el["scrap_date"]=gmt.localize(el["scrap_date"])
-        el["scrap_date"]=el["scrap_date"].astimezone(est)
-        el["scrap_date"]=el["scrap_date"].strftime(fmt)
+        el["scrap_date"] = gmt.localize(el["scrap_date"])
+        el["scrap_date"] = el["scrap_date"].astimezone(est)
+        el["scrap_date"] = el["scrap_date"].strftime(fmt)
         return el
+
     def jsonable(self, el):
         if el is None or "_id" not in el:
             return el
 
-        el["_id"]=str(el["_id"])
+        el["_id"] = str(el["_id"])
         return el
 
-
     def filter_articles(self, filter, limit=80):
-        db_filter={}
-        db_filter["matches"]=True
-        db_filter["deleted"]=False
-        db_filter["scrap_date"]={}
-        if "maxdate" in filter and len(filter["maxdate"])>7:
-            db_filter["scrap_date"]["$lt"]=datetime.datetime.strptime(filter["maxdate"],"%Y-%m-%d")
-
-        if "mindate" in filter and len(filter["mindate"])>7:
-            db_filter["scrap_date"]["$gt"]=datetime.datetime.strptime(filter["mindate"],"%Y-%m-%d")
-
-        if db_filter["scrap_date"]=={}:
+        db_filter = {}
+        db_filter["matches"] = True
+        db_filter["deleted"] = False
+        db_filter["scrap_date"] = {}
+        if "maxdate" in filter and len(filter["maxdate"]) > 7:
+            db_filter["scrap_date"]["$lt"] = datetime.datetime.strptime(
+                filter["maxdate"], "%Y-%m-%d")
+
+        if "mindate" in filter and len(filter["mindate"]) > 7:
+            db_filter["scrap_date"]["$gt"] = datetime.datetime.strptime(
+                filter["mindate"], "%Y-%m-%d")
+
+        if db_filter["scrap_date"] == {}:
             del db_filter["scrap_date"]
 
         if "title" in filter and filter["title"] != "":
-            filter["title"]=filter["title"].replace("(","\(").replace(")","\)")
-            db_filter["title"]={"$regex":".*%s.*"%filter["title"], "$options": 'i'}
+            filter["title"] = filter["title"].replace(
+                "(", "\(").replace(")", "\)")
+            db_filter["title"] = {"$regex": ".*%s.*" %
+                                  filter["title"], "$options": 'i'}
 
         if "site" in filter and filter["site"] != "":
-            filter["site"]=filter["site"].replace("(","\(").replace(")","\)")
-            db_filter["source_name"]={"$regex":".*%s.*"%filter["site"], "$options": 'i'}
-            #db_filter["$or"]=[
+            filter["site"] = filter["site"].replace(
+                "(", "\(").replace(")", "\)")
+            db_filter["source_name"] = {
+                "$regex": ".*%s.*" % filter["site"], "$options": 'i'}
+            # db_filter["$or"]=[
             #    {"source_name": {"$regex":".*%s.*"%filter["site"], "$options": 'i'}},
             #    {"url": {"$regex":".*%s.*"%filter["site"], "$options": 'i' }}
             #    ]
 
         if "minweight" not in filter or not filter["minweight"].isdigit():
-            filter["minweight"]=1
+            filter["minweight"] = 1
 
-        if "category" not in filter or filter["category"]=="-1":
-            filter["category"]=-1
+        if "category" not in filter or filter["category"] == "-1":
+            filter["category"] = -1
 
-        if int(filter["category"])>=0:
-            db_filter["category"]=int(filter["category"])
-            #pass
+        if int(filter["category"]) >= 0:
+            db_filter["category"] = int(filter["category"])
+            # pass
 
-        db_filter["weight"]={"$gt":int(filter["minweight"])}
+        db_filter["weight"] = {"$gt": int(filter["minweight"])}
 
-        if "keyword" in filter and len(filter["keyword"]) >0:
-            db_filter["kw"]={"$all": filter["keyword"]}
+        if "keyword" in filter and len(filter["keyword"]) > 0:
+            db_filter["kw"] = {"$all": filter["keyword"]}
 
-        #db_filter["kw.0"]={"$exists":False}
-        pageNumber=0
+        # db_filter["kw.0"]={"$exists":False}
+        pageNumber = 0
         if "page" in filter:
-            pageNumber=filter["page"]
-        #print(db_filter)
-        res = self.c_articles.find(db_filter,{"html":0,"text":0})
+            pageNumber = filter["page"]
+        # print(db_filter)
+        res = self.c_articles.find(db_filter, {"html": 0, "text": 0})
         if res is not None:
-            toSkip=(((pageNumber-1)*limit) if pageNumber > 0 else 0)
-            q = res.skip(toSkip).sort("scrap_date",DESCENDING).limit(limit)
+            toSkip = (((pageNumber - 1) * limit) if pageNumber > 0 else 0)
+            q = res.skip(toSkip).sort("scrap_date", DESCENDING).limit(limit)
             count = q.count()
         else:
             q = []
             count = 0
-        return { 'articles': [ self.jsonable(self.change_timezone(p)) for p in q ],
-                 'count' : count }
+        return {'articles': [self.jsonable(self.change_timezone(p)) for p in q],
+                'count': count}
 
     def article(self, id):
-        ret=self.c_articles.find_one({"_id":ObjectId(id)})
+        ret = self.c_articles.find_one({"_id": ObjectId(id)})
         return self.jsonable(ret)
 
-    def delete_article(self,artid):
-        return self.c_articles.update({"_id":ObjectId(artid)},{"$set":{"deleted":True}})
+    def delete_article(self, artid):
+        return self.c_articles.update({"_id": ObjectId(artid)}, {"$set": {"deleted": True}})
 
-    def undelete_article(self,artid):
-        return self.c_articles.update({"_id":ObjectId(artid)},{"$set":{"deleted":False}})
+    def undelete_article(self, artid):
+        return self.c_articles.update({"_id": ObjectId(artid)}, {"$set": {"deleted": False}})
 
-    def sources(self,id=None):
+    def sources(self, id=None):
         if id is None:
-            return [ self.jsonable(p) for p in self.c_sources.find() ]
+            return [self.jsonable(p) for p in self.c_sources.find()]
 
-        ret=self.c_sources.find_one({"_id":ObjectId(id)})
+        ret = self.c_sources.find_one({"_id": ObjectId(id)})
         return self.jsonable(ret)
 
     def rss_sources(self):
-        return [ p for p in self.c_sources.find({"rss":1}) ]
-
+        return [p for p in self.c_sources.find({"rss": 1})]
 
-    def insert_article(self,o):
+    def insert_article(self, o):
         try:
             self.c_articles.insert_one(o)
             return True
@@ -116,47 +123,45 @@ class db():
             return False
 
     def purge_error(self):
-        self.c_articles.remove({'error':True})
+        self.c_articles.remove({'error': True})
 
-    def article_exists(self,url):
-        ret=self.c_articles.find_one({'lowerurl':url.lower()})
+    def article_exists(self, url):
+        ret = self.c_articles.find_one({'lowerurl': url.lower()})
         if ret is None:
             return False
-        return True 
+        return True
 
-    def lower_arr(self,arr):
+    def lower_arr(self, arr):
         return [a.lower() for a in arr]
 
-    def to_regex(self,arr): #"ACH" |  "PAYMENT" | "EBAY"
-        return [ re.compile('\\b'+w.lower()+'\\b') for w in arr ]
+    def to_regex(self, arr):  # "ACH" |  "PAYMENT" | "EBAY"
+        return [re.compile('\\b' + w.lower() + '\\b') for w in arr]
 
     def get_keywords_list(self):
-        kw=self.client[self.DB]["keywords"]
-        return [  k["keyword"] for k in kw.find({}) ]
+        kw = self.client[self.DB]["keywords"]
+        return [k["keyword"] for k in kw.find({})]
 
-    def get_keywords(self,category):
-        kw=self.client[self.DB]["keywords"] #FIXME, split on "|" or regex
-        ret = [ {"weight":k["weight"],
+    def get_keywords(self, category):
+        kw = self.client[self.DB]["keywords"]  # FIXME, split on "|" or regex
+        ret = [{"weight": k["weight"],
                 "kw": k["keyword"],
-                "pattern": self.to_regex(k["keyword"].split("|")) } for k in  kw.find({}) ]#FIXME, categories
+                "pattern": self.to_regex(k["keyword"].split("|"))} for k in kw.find({})]  # FIXME, categories
         return ret
 
-    def add_source(self,url,rss,selector,name,category):
-        self.c_sources.insert({"link":url,
-                                "rss":rss,
-                                "selector":selector,
-                                "name":name,
-                                "category":category})
+    def add_source(self, url, rss, selector, name, category):
+        self.c_sources.insert({"link": url,
+                               "rss": rss,
+                               "selector": selector,
+                               "name": name,
+                               "category": category})
 
-
-    def reparse_articles(self,articles,kw):
+    def reparse_articles(self, articles, kw):
         for a in articles:
-            #print(a)
-            art=self.c_articles.find_one({"url":a["url"]})
-            res=news.match_keywords(kw,art["text"])
-            #(matches,w_sum,matching) 
-            #if(res["matches"]):
+            # print(a)
+            art = self.c_articles.find_one({"url": a["url"]})
+            res = news.match_keywords(kw, art["text"])
+            #(matches,w_sum,matching)
+            # if(res["matches"]):
             print(res)
-            self.c_articles.update({"_id":ObjectId(a["_id"])}, {"$set": res })
+            self.c_articles.update({"_id": ObjectId(a["_id"])}, {"$set": res})
             #matches, kw, weight
-

frontpage.py

@@ -4,35 +4,36 @@ import requests
 
 
 class FrontPage():
-    def getPage(self,url):
+
+    def getPage(self, url):
         r = requests.get(url)
         return r.content
-    
-    def getDomain(self,url):
+
+    def getDomain(self, url):
         parts = url.split('//', 1)
-        return parts[0]+'//'+parts[1].split('/', 1)[0]
-    
-    def fixHref(self,url,baseurl):
+        return parts[0] + '//' + parts[1].split('/', 1)[0]
+
+    def fixHref(self, url, baseurl):
         if url.startswith("http"):
             return url.strip()
 
-        if url.startswith("/"): #'abs'
-            domain=self.getDomain(baseurl)
+        if url.startswith("/"):  # 'abs'
+            domain = self.getDomain(baseurl)
             if domain.endswith("/") and url.startswith("/"):
-                url=url[1:]
-            url=domain+url
-        else: #relative
-            arr=baseurl.split("/")
+                url = url[1:]
+            url = domain + url
+        else:  # relative
+            arr = baseurl.split("/")
             arr.pop()
             arr.append(url)
-            url="/".join(arr)
+            url = "/".join(arr)
 
         return url.strip()
 
-    def __init__(self,url,selector):
+    def __init__(self, url, selector):
         soup = BeautifulSoup(self.getPage(url), 'html.parser')
-        links=soup.select(selector)
-        self.links=[]
+        links = soup.select(selector)
+        self.links = []
         for l in links:
             if not l.has_attr("href"):
                 print("PANIC")
@@ -41,4 +42,4 @@ class FrontPage():
                 continue
             #self.links.append({ "href": self.fixHref(l["href"],url),"title":l.get_text() })
             #self.links.append({ "href": self.fixHref(l["href"],url)})
-            self.links.append(self.fixHref(l["href"],url))
+            self.links.append(self.fixHref(l["href"], url))

functions.py

@@ -1,16 +1,17 @@
 import re
-unsharer=re.compile(r"(\?share=|#).+$",flags=re.IGNORECASE)
+unsharer = re.compile(r"(\?share=|#).+$", flags=re.IGNORECASE)
+
+
 def sanitizeUrl(url):
     global unsharer
-    ret = re.sub(unsharer, "",url)
+    ret = re.sub(unsharer, "", url)
     return ret
 
+
 def sanitizeSelector(s):
-    s=s.replace(">a","> a")
-    if re.match(r'=\w+\]',s) is None:
+    s = s.replace(">a", "> a")
+    if re.match(r'=\w+\]', s) is None:
         return s
 
-    ret=re.sub(r"=(\w+)]", r'="\1"]', s)
+    ret = re.sub(r"=(\w+)]", r'="\1"]', s)
     return ret
-
-

linklist.py

@@ -2,36 +2,39 @@ import feedparser
 from frontpage import FrontPage
 from functions import sanitizeUrl
 import newspaper
+
+
 class LinkList():
-    def __init__(self,url,selector=None,rss=False):
-        self.links=[]
+
+    def __init__(self, url, selector=None, rss=False):
+        self.links = []
         if rss:
             feed = feedparser.parse(url)
             if feed["bozo"]:
                 print(url)
                 print("RSS ERROR. PANIC")
-            #print(feed["bozo"]) FIXME: If bozo==1 => error
-            self.links=[i["link"] for i in feed["items"]]
+            # print(feed["bozo"]) FIXME: If bozo==1 => error
+            self.links = [i["link"] for i in feed["items"]]
         else:
 
-            if selector is None or selector =="":
-                #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida
-                s = newspaper.Source(url,memoize_articles=False, request_timeout=10)
+            if selector is None or selector == "":
+                # s=newspaper.build(url,language="en",memoize_articles=False)
+                # #memoize puede salvarme la vida
+                s = newspaper.Source(
+                    url, memoize_articles=False, request_timeout=10)
                 s.download()
                 s.parse()
                 s.set_categories()
                 s.download_categories()
                 s.parse_categories()
                 s.generate_articles()
-    
-                self.links=[a.url for a in s.articles] 
-            else:
-                f=FrontPage(url,selector)
-                self.links=f.links
-        self.links=self.purgeLinks(self.links)
-        self.links=list(set(self.links)) #avoid dupes
-
-    def purgeLinks(self,l):
-        return [ sanitizeUrl(link) for link in l if not "presslist" in link.lower() and not "videolist" in link.lower() ]
 
+                self.links = [a.url for a in s.articles]
+            else:
+                f = FrontPage(url, selector)
+                self.links = f.links
+        self.links = self.purgeLinks(self.links)
+        self.links = list(set(self.links))  # avoid dupes
 
+    def purgeLinks(self, l):
+        return [sanitizeUrl(link) for link in l if not "presslist" in link.lower() and not "videolist" in link.lower()]

news.py

@@ -4,47 +4,49 @@ import datetime
 import html
 
 from functions import sanitizeUrl
-from newspaper import Article,Config
+from newspaper import Article, Config
+
 
 class News():
-    r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
-    r_noise=re.compile(r"posted|Product|Google",flags=re.IGNORECASE)
-    
-    URL=""
-    LOWERURL=""
-    AUTHORS=[]
-    SCRAP_DATE=None 
-    PUBLISH_DATE=None
-    SUMMARY=""
-    TEXT=""
-    HTML=""
-    ERROR=False
-    SOURCE=""
-    SOURCENAME=""
-    WEIGHT_SUM=0
-    MATCHES=False
-    MATCHING_KW=[]
-    TITLE=""
-    CATEGORY=0
-
-    def __init__(self,url,sourceid, sourcename,sourcecategory,kw,lang="en",html=None):
-        url=sanitizeUrl(url)
-        self.URL=url
-        self.LOWERURL=url.lower()
-        self.SOURCENAME=sourcename
-        self.SOURCE=sourceid
-        self.CATEGORY=sourcecategory
-        self.MATCHING_KW=[]
+    r = re.compile(r"hours? ago|yesterday|today|last week|month",
+                   flags=re.IGNORECASE)
+    r_noise = re.compile(r"posted|Product|Google", flags=re.IGNORECASE)
+
+    URL = ""
+    LOWERURL = ""
+    AUTHORS = []
+    SCRAP_DATE = None
+    PUBLISH_DATE = None
+    SUMMARY = ""
+    TEXT = ""
+    HTML = ""
+    ERROR = False
+    SOURCE = ""
+    SOURCENAME = ""
+    WEIGHT_SUM = 0
+    MATCHES = False
+    MATCHING_KW = []
+    TITLE = ""
+    CATEGORY = 0
+
+    def __init__(self, url, sourceid, sourcename, sourcecategory, kw, lang="en", html=None):
+        url = sanitizeUrl(url)
+        self.URL = url
+        self.LOWERURL = url.lower()
+        self.SOURCENAME = sourcename
+        self.SOURCE = sourceid
+        self.CATEGORY = sourcecategory
+        self.MATCHING_KW = []
 
         config = Config()
-        config.language=lang
-        config.memoize_articles=False
-        config.keep_article_html=True
-        config.request_timeout=15
-        config.fetch_images=False
-        config.browser_user_agent="Mozilla/5.0 (X11; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"
-
-        a=None
+        config.language = lang
+        config.memoize_articles = False
+        config.keep_article_html = True
+        config.request_timeout = 15
+        config.fetch_images = False
+        config.browser_user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"
+
+        a = None
         try:
             a = Article(url=url, config=config)
             a.download(html=html)
@@ -58,83 +60,85 @@ class News():
             print(sourcename)
             if html is not None:
                 print(len(html))
-            self.ERROR=True
-            self.MATCHES=False
+            self.ERROR = True
+            self.MATCHES = False
             return
 
-        a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
-        
-        self.SCRAP_DATE=datetime.datetime.now()
-        self.AUTHORS=a.authors
-        self.PUBLISH_DATE=a.publish_date
-        self.SUMMARY=a.summary
-        self.TEXT=a.text
-        self.TITLE=self.fix_title(a.title)
-        self.HTML=htmlmin.minify(a.article_html,remove_empty_space=True)
+        a.authors = self.dedup([self.fix_author(
+            value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
+
+        self.SCRAP_DATE = datetime.datetime.now()
+        self.AUTHORS = a.authors
+        self.PUBLISH_DATE = a.publish_date
+        self.SUMMARY = a.summary
+        self.TEXT = a.text
+        self.TITLE = self.fix_title(a.title)
+        self.HTML = htmlmin.minify(a.article_html, remove_empty_space=True)
 #        if html is None:
 #            self.HTML=htmlmin.minify(a.article_html,remove_empty_space=True)
 #        else:
 #            self.HTML=htmlmin.minify(html,remove_empty_space=True)
-        self.ERROR=False
-        lowertext=self.TEXT.lower()
-        ret=match_keywords(kw,lowertext)
-        self.MATCHES=ret["matches"]
-        self.WEIGHT_SUM=ret["weight"]
-        self.MATCHING_KW=ret["kw"]
+        self.ERROR = False
+        lowertext = self.TEXT.lower()
+        ret = match_keywords(kw, lowertext)
+        self.MATCHES = ret["matches"]
+        self.WEIGHT_SUM = ret["weight"]
+        self.MATCHING_KW = ret["kw"]
 
     def get(self):
         return {
-                "url": self.URL,
-                "lowerurl": self.LOWERURL,
-                "source": self.SOURCE,
-                "source_name": self.SOURCENAME,
-                "authors": self.AUTHORS,
-                "scrap_date": self.SCRAP_DATE,
-                "publish_date": self.PUBLISH_DATE,
-                "summary": self.SUMMARY,
-                "text": self.TEXT,
-                "matches": self.MATCHES,
-                "weight": self.WEIGHT_SUM,
-                "kw": self.MATCHING_KW,
-                "html": self.HTML, 
-                "error": self.ERROR,
-                "deleted": False,
-                "category": self.CATEGORY,
-                "title": self.TITLE
-                }
-
-    def fix_title(self,n):
-        while(html.unescape(n)!=n):
-            n=html.unescape(n)
-        n=n.replace("&Dquot",'"')
+            "url": self.URL,
+            "lowerurl": self.LOWERURL,
+            "source": self.SOURCE,
+            "source_name": self.SOURCENAME,
+            "authors": self.AUTHORS,
+            "scrap_date": self.SCRAP_DATE,
+            "publish_date": self.PUBLISH_DATE,
+            "summary": self.SUMMARY,
+            "text": self.TEXT,
+            "matches": self.MATCHES,
+            "weight": self.WEIGHT_SUM,
+            "kw": self.MATCHING_KW,
+            "html": self.HTML,
+            "error": self.ERROR,
+            "deleted": False,
+            "category": self.CATEGORY,
+            "title": self.TITLE
+        }
+
+    def fix_title(self, n):
+        while(html.unescape(n) != n):
+            n = html.unescape(n)
+        n = n.replace("&Dquot", '"')
         return n
 
-    def dedup(self,val):
+    def dedup(self, val):
         return list(set(val))
 
-    def isNoise(self,v):
+    def isNoise(self, v):
         if self.r_noise.match(v):
             return True
         return False
-    def isComment(self,v):
+
+    def isComment(self, v):
         if self.r.match(v):
             return True
         return False
-    
-    def fix_author(self,a):
+
+    def fix_author(self, a):
         return a.replace("_", " ").lower()
 
 
 def match_keywords(keywords, lowertext):
-    w_sum=0
-    matching=[]
-    matches=False
+    w_sum = 0
+    matching = []
+    matches = False
     for k in keywords:
-        num_matches=len( [ 1 for kwp in k["pattern"] if kwp.search(lowertext) ] )
-        if num_matches==len(k["pattern"]):
-            w_sum+=k["weight"]
+        num_matches = len([1 for kwp in k["pattern"] if kwp.search(lowertext)])
+        if num_matches == len(k["pattern"]):
+            w_sum += k["weight"]
             matching.append(k["kw"])
-            matches=True
+            matches = True
 
-    ret=({"matches":matches,"weight":w_sum,"kw":matching})
+    ret = ({"matches": matches, "weight": w_sum, "kw": matching})
     return ret

paid_parser.py

@@ -3,12 +3,14 @@ import paid_sources.financial_times
 import paid_sources.wsj
 import paid_sources.forbes
 
+
 class PaidParser:
-    html=""
-    def __init__(self,url):
+    html = ""
+
+    def __init__(self, url):
         if "ft.com" in url:
-            self.html=paid_sources.financial_times.parse_ft(url)
+            self.html = paid_sources.financial_times.parse_ft(url)
         if "wsj.com" in url:
-            self.html=paid_sources.wsj.parse(url)
+            self.html = paid_sources.wsj.parse(url)
         if "forbes.com" in url:
-            self.html=paid_sources.forbes.parse(url)
+            self.html = paid_sources.forbes.parse(url)

reparse.py

@@ -2,10 +2,10 @@
 
 from db import db
 
-d=db()
-kw=d.get_keywords(0)
+d = db()
+kw = d.get_keywords(0)
 
-arts=d.filter_articles({"kw.0":{"$exists":False}},limit=8000)
-#print(arts)
+arts = d.filter_articles({"kw.0": {"$exists": False}}, limit=8000)
+# print(arts)
 print(len(arts["articles"]))
-d.reparse_articles(arts["articles"],kw)
+d.reparse_articles(arts["articles"], kw)

source.py

@@ -1,17 +1,16 @@
 
 class Source():
-    URL=""
-    BRAND=""
-    DESCRIPTION=""
-    CATEGORY=0
-    def __init__(self,url,category,lang="en"):
-        s=newspaper.build(url,language=lang,memoize_articles=False) 
-        self.BRAND=s.brand
-        self.URL=url
-        self.DESCRIPTION=s.description
-        self.CATEGORY=category
-
-    def obj(self):
-        return {"url":self.URL, "brand": self.BRAND, "desc": self.DESCRIPTION, "category":self.CATEGORY}
+    URL = ""
+    BRAND = ""
+    DESCRIPTION = ""
+    CATEGORY = 0
 
+    def __init__(self, url, category, lang="en"):
+        s = newspaper.build(url, language=lang, memoize_articles=False)
+        self.BRAND = s.brand
+        self.URL = url
+        self.DESCRIPTION = s.description
+        self.CATEGORY = category
 
+    def obj(self):
+        return {"url": self.URL, "brand": self.BRAND, "desc": self.DESCRIPTION, "category": self.CATEGORY}

test_article.py

@@ -1,24 +1,24 @@
 import newspaper
-from newspaper import Article,Config
+from newspaper import Article, Config
 import sys
 import requests
 
 url = sys.argv[1]
 
 config = Config()
-config.language='en'
-config.memoize_articles=False
-config.keep_article_html=True
-config.request_timeout=10
-config.fetch_images=False
-config.browser_user_agent="Mozilla/5.0 (X11; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"
-config.verbose=True
+config.language = 'en'
+config.memoize_articles = False
+config.keep_article_html = True
+config.request_timeout = 10
+config.fetch_images = False
+config.browser_user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"
+config.verbose = True
 
 
 #r = requests.get(url)
-#print(r.status_code)
-#print(r.text)
-a = Article(url=url,config=config)
+# print(r.status_code)
+# print(r.text)
+a = Article(url=url, config=config)
 print(url)
 a.download()
 print("dl")

web.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-from flask import Flask,abort,jsonify,request
+from flask import Flask, abort, jsonify, request
 from db import db
 import json
 from linklist import LinkList
@@ -9,86 +9,94 @@ app = Flask(__name__)
 d = db()
 
 
-BASEPATH="/pymnts"
-@app.route(BASEPATH+'/')
+BASEPATH = "/pymnts"
+
+
+@app.route(BASEPATH + '/')
 def index():
     return "Hello, World!"
 
-@app.route(BASEPATH+'/source/add/', methods=['POST'])
+
+@app.route(BASEPATH + '/source/add/', methods=['POST'])
 def add_source():
-    selector=None
-    rss=False
-    json=request.get_json()
+    selector = None
+    rss = False
+    json = request.get_json()
     if "selector" in json:
-        selector=json["selector"]
+        selector = json["selector"]
     if "rss" in json:
-        rss=json["rss"]
+        rss = json["rss"]
 
     if "category" not in json:
-        json["category"]=1
+        json["category"] = 1
 
-    
-    json["category"]=int(json["category"])
-    d.add_source(json["source"],rss,selector,json["name"],json["category"])
-    return jsonify({'ok': "ok"}) 
+    json["category"] = int(json["category"])
+    d.add_source(json["source"], rss, selector, json["name"], json["category"])
+    return jsonify({'ok': "ok"})
 
-@app.route(BASEPATH+'/source/test/', methods=['POST'])
+
+@app.route(BASEPATH + '/source/test/', methods=['POST'])
 def test_source():
-    selector=None
-    rss=False
-    json=request.get_json()
-    if "selector" in json and len(json["selector"]) >0:
-        selector=json["selector"]
+    selector = None
+    rss = False
+    json = request.get_json()
+    if "selector" in json and len(json["selector"]) > 0:
+        selector = json["selector"]
     if "rss" in json:
-        rss=json["rss"]
+        rss = json["rss"]
+
+    l = LinkList(json["source"], selector, rss=rss)
+    return jsonify({'links': l.links})
 
-    l=LinkList(json["source"],selector,rss=rss)
-    return jsonify({'links': l.links}) 
 
-@app.route(BASEPATH+'/articles/', methods=['POST'])
-@app.route(BASEPATH+'/articles', methods=['POST'])
+@app.route(BASEPATH + '/articles/', methods=['POST'])
+@app.route(BASEPATH + '/articles', methods=['POST'])
 def filter_articles():
-    return jsonify(d.filter_articles(request.get_json(),limit=15)) 
+    return jsonify(d.filter_articles(request.get_json(), limit=15))
 
 
-@app.route(BASEPATH+'/keywords/', methods=['GET'])
-@app.route(BASEPATH+'/keywords', methods=['GET'])
+@app.route(BASEPATH + '/keywords/', methods=['GET'])
+@app.route(BASEPATH + '/keywords', methods=['GET'])
 def get_keywords():
-    return jsonify({'keywords': d.get_keywords_list()}) #limit
+    return jsonify({'keywords': d.get_keywords_list()})  # limit
 
-@app.route(BASEPATH+'/articles/', methods=['GET'])
-@app.route(BASEPATH+'/articles', methods=['GET'])
+
+@app.route(BASEPATH + '/articles/', methods=['GET'])
+@app.route(BASEPATH + '/articles', methods=['GET'])
 def get_articles():
-    return jsonify(d.filter_articles({},limit=15)) 
+    return jsonify(d.filter_articles({}, limit=15))
 
 
-@app.route(BASEPATH+'/article/delete/<string:art_id>', methods=['GET'])
+@app.route(BASEPATH + '/article/delete/<string:art_id>', methods=['GET'])
 def delete_article(art_id):
     if not d.delete_article(art_id):
         abort(401)
-    return jsonify({'ok':"ok"})
+    return jsonify({'ok': "ok"})
+
 
-@app.route(BASEPATH+'/article/<string:art_id>', methods=['GET'])
+@app.route(BASEPATH + '/article/<string:art_id>', methods=['GET'])
 def get_article(art_id):
-    art=d.article(art_id)
+    art = d.article(art_id)
     if art is None:
         abort(401)
     return jsonify({'articles': art})
 
-@app.route(BASEPATH+'/sources/', methods=['GET'])
-@app.route(BASEPATH+'/sources', methods=['GET'])
+
+@app.route(BASEPATH + '/sources/', methods=['GET'])
+@app.route(BASEPATH + '/sources', methods=['GET'])
 def get_sources():
     return jsonify({'sources': sorted(d.sources(), key=lambda k: k['name'].lower())})
 
-@app.route(BASEPATH+'/sources/<string:source_id>', methods=['GET'])
+
+@app.route(BASEPATH + '/sources/<string:source_id>', methods=['GET'])
 def get_source(source_id):
-    source=d.sources(source_id)
+    source = d.sources(source_id)
     if source is None:
         abort(401)
     return jsonify({'source': source})
 
 
-@app.route(BASEPATH+'/add_source', methods=['POST'])
+@app.route(BASEPATH + '/add_source', methods=['POST'])
 def create_task():
     if not request.json or not 'title' in request.json:
         abort(400)
@@ -102,4 +110,4 @@ def create_task():
     return jsonify({'task': task}), 201
 
 if __name__ == '__main__':
-    app.run(host="0.0.0.0",debug=True)
+    app.run(host="0.0.0.0", debug=True)