alexis2/linklist.py

import feedparser
from frontpage import FrontPage
from functions import sanitizeUrl
import newspaper
class LinkList():
    def __init__(self,url,selector=None,rss=False):
        self.links=[]
        if rss:
            feed = feedparser.parse(url)
            if feed["bozo"]:
                print("RSS ERROR. PANIC")
            #print(feed["bozo"]) FIXME: If bozo==1 => error
            self.links=[i["link"] for i in feed["items"]]
            print(self.links)
        else:
            if selector is None:
                #s=newspaper.build(url,language="en",memoize_articles=False) #memoize puede salvarme la vida
                s = newspaper.Source(url,memoize_articles=False)
                s.download()
                s.parse()
                s.set_categories()
                s.download_categories()
                s.parse_categories()
                s.generate_articles()
    
                self.links=[a.url for a in s.articles] 
            else:
                f=FrontPage(url,selector)
                self.links=f.links
        self.links=self.purgeLinks(self.links)
        self.links=list(set(self.links)) #avoid dupes

    def purgeLinks(self,l):
        return [ sanitizeUrl(link) for link in l if not "presslist" in link.lower() and not "videolist" in link.lower() ]