| 123456789101112131415161718192021222324252627282930313233343536373839404142434445 |
- #!/usr/bin/python3
- from bs4 import BeautifulSoup
- import requests
- class FrontPage():
- def getPage(self, url):
- r = requests.get(url)
- return r.content
- def getDomain(self, url):
- parts = url.split('//', 1)
- return parts[0] + '//' + parts[1].split('/', 1)[0]
- def fixHref(self, url, baseurl):
- if url.startswith("http"):
- return url.strip()
- if url.startswith("/"): # 'abs'
- domain = self.getDomain(baseurl)
- if domain.endswith("/") and url.startswith("/"):
- url = url[1:]
- url = domain + url
- else: # relative
- arr = baseurl.split("/")
- arr.pop()
- arr.append(url)
- url = "/".join(arr)
- return url.strip()
- def __init__(self, url, selector):
- soup = BeautifulSoup(self.getPage(url), 'html.parser')
- links = soup.select(selector)
- self.links = []
- for l in links:
- if not l.has_attr("href"):
- print("PANIC")
- print(l)
- print(url)
- continue
- #self.links.append({ "href": self.fixHref(l["href"],url),"title":l.get_text() })
- #self.links.append({ "href": self.fixHref(l["href"],url)})
- self.links.append(self.fixHref(l["href"], url))
|
