|
|
@@ -60,6 +60,7 @@ class LinkList():
|
|
|
|
|
|
class News():
|
|
|
r=re.compile(r"hours? ago|yesterday|today|last week|month",flags=re.IGNORECASE)
|
|
|
+ r_noise=re.compile(r"posted|Product|Google",flags=re.IGNORECASE)
|
|
|
def __init__(self,url,lang="en"):
|
|
|
url=self.sanitizeUrl(url)
|
|
|
a = Article(url, language=lang,keep_article_html=True)
|
|
|
@@ -67,7 +68,7 @@ class News():
|
|
|
a.parse()
|
|
|
a.nlp()
|
|
|
|
|
|
- a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value)])
|
|
|
+ a.authors=self.dedup([self.fix_author(value) for value in a.authors if not self.isComment(value) and not self.isNoise(value)])
|
|
|
|
|
|
print(a.authors)
|
|
|
print(a.publish_date)
|
|
|
@@ -85,6 +86,10 @@ class News():
|
|
|
def dedup(self,val):
|
|
|
return list(set(val))
|
|
|
|
|
|
+ def isNoise(self,v):
|
|
|
+ if self.r_noise.match(v):
|
|
|
+ return True
|
|
|
+ return False
|
|
|
def isComment(self,v):
|
|
|
if self.r.match(v):
|
|
|
return True
|
|
|
@@ -94,21 +99,34 @@ class News():
|
|
|
return a.replace("_", " ").lower()
|
|
|
|
|
|
|
|
|
+l=None
|
|
|
#d=db()
|
|
|
#l=LinkList("http://9to5google.com")
|
|
|
#l=LinkList("http://www.alibabagroup.com/en/news/press")
|
|
|
#l=LinkList("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-news", selector=".ccbnTblLnk")
|
|
|
#l=LinkList("http://about.americanexpress.com/news/pr/releases.aspx")
|
|
|
#NOT WORKING l=LinkList("http://www.americanbanker.com/")
|
|
|
-l=LinkList("http://officialandroid.blogspot.com/atom.xml",rss=True)
|
|
|
+#l=LinkList("http://officialandroid.blogspot.com/atom.xml",rss=True)
|
|
|
+#l=LinkList("https://www.apple.com/pr/feeds/pr.rss",rss=True)
|
|
|
+#l=LinkList("http://appleinsider.com/rss/news/",rss=True)
|
|
|
+#l=LinkList("http://newsroom.bankofamerica.com/feeds/press_release/all/rss.xml",rss=True)
|
|
|
+#l=LinkList("https://newsroom.fb.com/feed/",rss=True)
|
|
|
+#n=News("http://appleinsider.com/articles/16/05/11/apple-again-rumored-to-turn-la-landmark-into-retail-store")
|
|
|
#n=News("http://9to5google.com/2016/05/11/nest-opensource-openthread-networking-protocol/?share=google-plus-1")
|
|
|
#n=News("http://www.alibabagroup.com/en/news/article?news=p150908b")
|
|
|
#n=News("http://phx.corporate-ir.net/phoenix.zhtml?c=176060&p=irol-newsArticle&ID=1250170")
|
|
|
#n=News("http://about.americanexpress.com/news/pr/2016/amex-dividend-payable-august-10-2016.aspx")
|
|
|
+#n=News("http://www.apple.com/pr/library/2015/06/08Apple-Pay-Giving-Shoppers-Even-More-Ways-to-Pay.html")
|
|
|
+#n=News("http://newsroom.bankofamerica.com/press-releases/small-business-banking/bank-america-survey-finds-small-business-owners-confidence-dow")
|
|
|
+#n=News("http://newsroom.fb.com/news/2016/04/news-feed-fyi-more-articles-you-want-to-spend-time-viewing/")
|
|
|
+
|
|
|
+#l=LinkList("https://www.blogger.com/feeds/10861780/posts/default",rss=True)
|
|
|
+n=News("http://googleblog.blogspot.com/2016/05/translate-where-you-need-it-in-any-app.html")
|
|
|
+if l is not None:
|
|
|
+ for a in l.links:
|
|
|
+ print(a)
|
|
|
+ print(len(l.links))
|
|
|
|
|
|
-for a in l.links:
|
|
|
- print(a)
|
|
|
-print(len(l.links))
|
|
|
sys.exit(1)
|
|
|
def sanitizeSelector(s):
|
|
|
s=s.replace(">a","> a")
|
