|
|
@@ -24,24 +24,33 @@ class News():
|
|
|
MATCHING_KW=[]
|
|
|
TITLE=""
|
|
|
|
|
|
- def __init__(self,url,sourceid, sourcename,kw,lang="en"):
|
|
|
+ def __init__(self,url,sourceid, sourcename,kw,lang="en",html=None):
|
|
|
url=sanitizeUrl(url)
|
|
|
self.URL=url
|
|
|
self.SOURCENAME=sourcename
|
|
|
self.SOURCE=sourceid
|
|
|
self.MATCHING_KW=[]
|
|
|
|
|
|
- a = Article(url, language=lang,keep_article_html=True,request_timeout=10)
|
|
|
+ a=None
|
|
|
try:
|
|
|
- a.download()
|
|
|
+ if html is None:
|
|
|
+ a = Article(url, language=lang,keep_article_html=True,request_timeout=10)
|
|
|
+ a.download()
|
|
|
+ else:
|
|
|
+ a= Article("")
|
|
|
+ a.set_html(html)
|
|
|
+
|
|
|
a.parse()
|
|
|
a.nlp()
|
|
|
except Exception as e:
|
|
|
print("ERROR parsing/downloading/nlp article")
|
|
|
print(e)
|
|
|
+ if html is not None:
|
|
|
+ print(len(html))
|
|
|
self.ERROR=True
|
|
|
self.MATCHES=False
|
|
|
return
|
|
|
+
|
|
|
a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
|
|
|
|
|
|
self.SCRAP_DATE=datetime.datetime.now()
|
