|
|
@@ -1,146 +1,11 @@
|
|
|
#!/usr/bin/python3
|
|
|
-import sys
|
|
|
-import re
|
|
|
-import newspaper
|
|
|
-import pymongo
|
|
|
-import htmlmin
|
|
|
-import feedparser
|
|
|
-import datetime
|
|
|
import threading
|
|
|
-
|
|
|
from queue import Queue
|
|
|
-from frontpage import FrontPage
|
|
|
-from newspaper import Article
|
|
|
+
|
|
|
from db import db
|
|
|
from source import Source
|
|
|
-
|
|
|
-class LinkList():
|
|
|
- def __init__(self,url,selector=None,rss=False):
|
|
|
- self.links=[]
|
|
|
- if rss:
|
|
|
- feed = feedparser.parse(url)
|
|
|
- if feed["bozo"]:
|
|
|
- print("RSS ERROR. PANIC")
|
|
|
- #print(feed["bozo"]) FIXME: If bozo==1 => error
|
|
|
- self.links=[i["link"] for i in feed["items"]]
|
|
|
- print(self.links)
|
|
|
- else:
|
|
|
- if selector is None:
|
|
|
- #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida
|
|
|
- s = newspaper.Source(url,memoize_articles=False)
|
|
|
- s.download()
|
|
|
- s.parse()
|
|
|
- s.set_categories()
|
|
|
- s.download_categories()
|
|
|
- s.parse_categories()
|
|
|
- s.generate_articles()
|
|
|
-
|
|
|
- self.links=[a.url for a in s.articles]
|
|
|
- else:
|
|
|
- f=FrontPage(url,selector)
|
|
|
- self.links=f.links
|
|
|
- self.links=self.purgeLinks(self.links)
|
|
|
- self.links=list(set(self.links)) #avoid dupes
|
|
|
-
|
|
|
- def purgeLinks(self,l):
|
|
|
- return [ sanitizeUrl(link) for link in l if not "presslist" in link.lower() and not "videolist" in link.lower() ]
|
|
|
-
|
|
|
-class News():
|
|
|
- r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
|
|
|
- r_noise=re.compile(r"posted|Product|Google",flags=re.IGNORECASE)
|
|
|
-
|
|
|
- URL=""
|
|
|
- AUTHORS=[]
|
|
|
- SCRAP_DATE=None
|
|
|
- PUBLISH_DATE=None
|
|
|
- SUMMARY=""
|
|
|
- TEXT=""
|
|
|
- HTML=""
|
|
|
- ERROR=False
|
|
|
- SOURCE=""
|
|
|
- WEIGHT_SUM=0
|
|
|
- MATCHES=False
|
|
|
- MATCHING_KW=[]
|
|
|
-
|
|
|
- def __init__(self,url,source,kw,lang="en"):
|
|
|
- url=sanitizeUrl(url)
|
|
|
- self.URL=url
|
|
|
- self.SOURCE=source
|
|
|
-
|
|
|
- a = Article(url, language=lang,keep_article_html=True)
|
|
|
- try:
|
|
|
- a.download()
|
|
|
- a.parse()
|
|
|
- except newspaper.article.ArticleException:
|
|
|
- print("ERROR parsing article")
|
|
|
- ERROR=True
|
|
|
- return
|
|
|
- except Exception:
|
|
|
- print("ERROR downloading article")
|
|
|
- ERROR=True
|
|
|
- return
|
|
|
- a.nlp()
|
|
|
-
|
|
|
- a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
|
|
|
-
|
|
|
- self.SCRAP_DATE=datetime.datetime.now()
|
|
|
- self.AUTHORS=a.authors
|
|
|
- self.PUBLISH_DATE=a.publish_date
|
|
|
- self.SUMMARY=a.summary
|
|
|
- self.TEXT=a.text
|
|
|
- self.HTML=htmlmin.minify(a.article_html,remove_empty_space=True)
|
|
|
- for k in kw:
|
|
|
- if re.match(k["pattern"], self.TEXT):
|
|
|
- self.WEIGHT_SUM+=k["weight"]
|
|
|
- self.MATCHING_KW.append(k["kw"])
|
|
|
- self.MATCHES=True
|
|
|
-
|
|
|
-
|
|
|
- def get(self):
|
|
|
- return {
|
|
|
- "url": self.URL,
|
|
|
- "source": self.SOURCE,
|
|
|
- "authors": self.AUTHORS,
|
|
|
- "scrap_date": self.SCRAP_DATE,
|
|
|
- "publish_date": self.PUBLISH_DATE,
|
|
|
- "summary": self.SUMMARY,
|
|
|
- "text": self.TEXT,
|
|
|
- "matches": self.MATCHES,
|
|
|
- "weight": self.WEIGHT_SUM,
|
|
|
- "kw": self.MATCHING_KW,
|
|
|
- "html": self.HTML
|
|
|
- }
|
|
|
-
|
|
|
- def dedup(self,val):
|
|
|
- return list(set(val))
|
|
|
-
|
|
|
- def isNoise(self,v):
|
|
|
- if self.r_noise.match(v):
|
|
|
- return True
|
|
|
- return False
|
|
|
- def isComment(self,v):
|
|
|
- if self.r.match(v):
|
|
|
- return True
|
|
|
- return False
|
|
|
-
|
|
|
- def fix_author(self,a):
|
|
|
- return a.replace("_", " ").lower()
|
|
|
-
|
|
|
-
|
|
|
-unsharer=re.compile(r"(\?share=|#).+$",flags=re.IGNORECASE)
|
|
|
-def sanitizeUrl(url):
|
|
|
- global unsharer
|
|
|
- ret = re.sub(unsharer, "",url)
|
|
|
- return ret
|
|
|
-
|
|
|
-def sanitizeSelector(s):
|
|
|
- s=s.replace(">a","> a")
|
|
|
- if re.match(r'=\w+\]',s) is None:
|
|
|
- return s
|
|
|
-
|
|
|
- ret=re.sub(r"=(\w+)]", r'="\1"]', s)
|
|
|
- return ret
|
|
|
-
|
|
|
+from linklist import LinkList
|
|
|
+from news import News
|
|
|
|
|
|
def parseSource(s):
|
|
|
sel=None
|
