diff --git a/pdf.py b/pdf.py index d202682..a9baf4b 100644 --- a/pdf.py +++ b/pdf.py @@ -1,9 +1,9 @@ import re -import sys import os -import requests import argparse +from bs4 import BeautifulSoup from urllib.request import urlopen +import ssl ''' argparse to enable finicky command-line args @@ -14,32 +14,40 @@ parser.add_argument('relative_url', nargs='?', default='none') args = parser.parse_args() -# HTML Parse URL +# HTML Parse URL url_link = str(args.website_url) # wget URL wget_link = str(args.relative_url) # system path sys_path = str(args.file_path) -website = urlopen(url_link) +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE -# Decode website into string string -html = website.read().decode('utf-8') +website = urlopen(url_link, context=ctx) -#Find all local and non-hosted pdfs and download -links = re.findall('"(https?://\S*?.pdf)"', html) -local_links = re.findall('"([^.\"=]*.pdf)', html) +# Decode website into string string +soup = BeautifulSoup(website.read(), features="html.parser") +links = [] +local_links = [] +divs = soup.find_all('a', attrs={'href': re.compile(".pdf$")}) +for div in divs: + href = div.get('href') + if href.find("http") == -1: + local_links.append(href) + else: + links.append(href) -#Download non-hosted PDFs +# Download non-hosted PDFs for link in links: - command = 'wget' - os.system("%s %s -P %s" % (command, link, sys_path)) + command = 'wget' + os.system("%s %s -P %s" % (command, link, sys_path)) -#Download local/hosted PDFs +# Download local/hosted PDFs for link in local_links: - command = 'wget' - if(wget_link == 'none'): - os.system("%s %s/%s -P %s" % (command, url_link, link, sys_path)) - else: - os.system("%s %s/%s -P %s" % (command, wget_link, link, sys_path)) - \ No newline at end of file + command = 'wget' + if(wget_link == 'none'): + os.system("%s '%s/%s' -P %s" % (command, url_link, link, sys_path)) + else: + os.system("%s '%s/%s' -P %s" % (command, wget_link, link, sys_path))