| 1234567891011121314151617181920212223242526272829303132333435363738394041424344 |
- #!/usr/bin/python3
- from bs4 import BeautifulSoup
- import requests
- class FrontPage():
- def getPage(self,url):
- r = requests.get(url)
- return r.content
-
- def getDomain(self,url):
- parts = url.split('//', 1)
- return parts[0]+'//'+parts[1].split('/', 1)[0]
-
- def fixHref(self,url,baseurl):
- if url.startswith("http"):
- return url.strip()
- if url.startswith("/"): #'abs'
- domain=self.getDomain(baseurl)
- if domain.endswith("/") and url.startswith("/"):
- url=url[1:]
- url=domain+url
- else: #relative
- arr=baseurl.split("/")
- arr.pop()
- arr.append(url)
- url="/".join(arr)
- return url.strip()
- def __init__(self,url,selector):
- soup = BeautifulSoup(self.getPage(url), 'html.parser')
- links=soup.select(selector)
- self.links=[]
- for l in links:
- if not l.has_attr("href"):
- print("PANIC")
- print(l)
- print(url)
- continue
- #self.links.append({ "href": self.fixHref(l["href"],url),"title":l.get_text() })
- #self.links.append({ "href": self.fixHref(l["href"],url)})
- self.links.append(self.fixHref(l["href"],url))
|
