From 046e7124bad4401076ca64d1ee5235a6c6a49e81 Mon Sep 17 00:00:00 2001 From: wujiabang Date: Thu, 20 Jan 2022 21:57:36 +0800 Subject: [PATCH 1/2] do not check certificate --- pdf.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pdf.py b/pdf.py index d202682..0ee93bb 100644 --- a/pdf.py +++ b/pdf.py @@ -3,6 +3,7 @@ import os import requests import argparse +import ssl from urllib.request import urlopen ''' @@ -21,7 +22,11 @@ # system path sys_path = str(args.file_path) -website = urlopen(url_link) +ctx = ssl.create_default_context() +ctx.check_hostname = False +ctx.verify_mode = ssl.CERT_NONE + +website = urlopen(url_link,context=ctx) # Decode website into string string html = website.read().decode('utf-8') From 34d137d7b21f3e72d7828adb94f9fbef38bd44d1 Mon Sep 17 00:00:00 2001 From: wujiabang Date: Thu, 20 Jan 2022 22:56:50 +0800 Subject: [PATCH 2/2] fix parsing pdf file failure when file name contains white spaces --- pdf.py | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/pdf.py b/pdf.py index 0ee93bb..a9baf4b 100644 --- a/pdf.py +++ b/pdf.py @@ -1,10 +1,9 @@ import re -import sys import os -import requests import argparse -import ssl +from bs4 import BeautifulSoup from urllib.request import urlopen +import ssl ''' argparse to enable finicky command-line args @@ -15,7 +14,7 @@ parser.add_argument('relative_url', nargs='?', default='none') args = parser.parse_args() -# HTML Parse URL +# HTML Parse URL url_link = str(args.website_url) # wget URL wget_link = str(args.relative_url) @@ -26,25 +25,29 @@ ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE -website = urlopen(url_link,context=ctx) +website = urlopen(url_link, context=ctx) # Decode website into string string -html = website.read().decode('utf-8') - -#Find all local and non-hosted pdfs and download -links = re.findall('"(https?://\S*?.pdf)"', html) -local_links = re.findall('"([^.\"=]*.pdf)', html) - -#Download non-hosted PDFs +soup = BeautifulSoup(website.read(), features="html.parser") +links = [] +local_links = [] +divs = soup.find_all('a', attrs={'href': re.compile(".pdf$")}) +for div in divs: + href = div.get('href') + if href.find("http") == -1: + local_links.append(href) + else: + links.append(href) + +# Download non-hosted PDFs for link in links: - command = 'wget' - os.system("%s %s -P %s" % (command, link, sys_path)) + command = 'wget' + os.system("%s %s -P %s" % (command, link, sys_path)) -#Download local/hosted PDFs +# Download local/hosted PDFs for link in local_links: - command = 'wget' - if(wget_link == 'none'): - os.system("%s %s/%s -P %s" % (command, url_link, link, sys_path)) - else: - os.system("%s %s/%s -P %s" % (command, wget_link, link, sys_path)) - \ No newline at end of file + command = 'wget' + if(wget_link == 'none'): + os.system("%s '%s/%s' -P %s" % (command, url_link, link, sys_path)) + else: + os.system("%s '%s/%s' -P %s" % (command, wget_link, link, sys_path))