| 12345678910111213141516171819202122232425262728293031323334353637383940 |
- import feedparser
- from frontpage import FrontPage
- from functions import sanitizeUrl
- import newspaper
- class LinkList():
- def __init__(self, url, selector=None, rss=False):
- self.links = []
- if rss:
- feed = feedparser.parse(url)
- if feed["bozo"]:
- print(url)
- print("RSS ERROR. PANIC")
- # print(feed["bozo"]) FIXME: If bozo==1 => error
- self.links = [i["link"] for i in feed["items"] if "link" in i ]
- else:
- if selector is None or selector == "":
- # s=newspaper.build(url,language="en",memoize_articles=False)
- # #memoize puede salvarme la vida
- s = newspaper.Source(
- url, memoize_articles=False, request_timeout=10)
- s.download()
- s.parse()
- s.set_categories()
- s.download_categories()
- s.parse_categories()
- s.generate_articles()
- self.links = [a.url for a in s.articles]
- else:
- f = FrontPage(url, selector)
- self.links = f.links
- self.links = self.purgeLinks(self.links)
- self.links = list(set(self.links)) # avoid dupes
- def purgeLinks(self, l):
- return [sanitizeUrl(link) for link in l if not "presslist" in link.lower() and not "videolist" in link.lower()]
|
