From 74c4645d11846a4b6b402c926b6b913e4e3cbc14 Mon Sep 17 00:00:00 2001 From: eight04 Date: Thu, 2 Apr 2026 21:50:38 +0800 Subject: [PATCH 1/7] Bump deps --- requirements-lock.txt | 74 +++++++++++++++++++++---------------------- requirements.txt | 10 +++--- setup.cfg | 14 ++++---- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/requirements-lock.txt b/requirements-lock.txt index de4dd1b..82e4785 100644 --- a/requirements-lock.txt +++ b/requirements-lock.txt @@ -1,66 +1,66 @@ ansicon==1.89.0 -astroid==3.3.9 +astroid==4.0.4 belfrywidgets==1.0.3 bidict==0.23.1 -blessed==1.20.0 -Brotli==1.1.0 -certifi==2025.1.31 -cffi==1.17.1 -charset-normalizer==3.4.1 +blessed==1.38.0 +brotli==1.2.0 +certifi==2026.2.25 +cffi==2.0.0 +charset-normalizer==3.4.7 colorama==0.4.6 -curl_cffi==0.10.0 +curl_cffi==0.14.0 deno_vm==0.6.0 -desktop3==0.5.3 -dill==0.3.9 +desktop3==0.5.4 +dill==0.4.1 docopt==0.6.2 -docutils==0.21.2 +docutils==0.22.4 enlighten==1.14.1 -id==1.5.0 -idna==3.10 +id==1.6.1 +idna==3.11 importlib_metadata==8.5.0 -isort==6.0.1 +isort==8.0.1 jaraco.classes==3.4.0 -jaraco.context==6.0.1 -jaraco.functools==4.1.0 -jinxed==1.3.0 -keyring==25.6.0 +jaraco.context==6.1.2 +jaraco.functools==4.4.0 +jinxed==1.4.0 +keyring==25.7.0 livereload==2.7.1 -markdown-it-py==3.0.0 +markdown-it-py==4.0.0 mccabe==0.7.0 mdurl==0.1.2 -more-itertools==10.6.0 -nh3==0.2.21 +more-itertools==10.8.0 +nh3==0.3.4 ordered-set==3.1.1 -packaging==24.2 +packaging==26.0 pkginfo==1.10.0 -platformdirs==4.3.6 +platformdirs==4.9.4 prefixed==0.9.0 -puremagic==1.28 -pycparser==2.22 -pycryptodomex==3.21.0 -Pygments==2.19.1 -pylint==3.3.5 -pyperclip==1.9.0 +puremagic==2.1.1 +pycparser==3.0 +pycryptodomex==3.23.0 +Pygments==2.20.0 +pylint==4.0.5 +pyperclip==1.11.0 pythreadworker==0.10.0 pywin32-ctypes==0.2.3 pyxcute==0.8.1 readme_renderer==44.0 -requests==2.32.3 +requests==2.33.1 requests-toolbelt==1.0.0 rfc3986==2.0.0 -rich==13.9.4 +rich==14.3.3 safeprint==0.2.0 semver==2.13.0 Send2Trash==1.8.3 -setuptools==76.0.0 +setuptools==82.0.1 six==1.17.0 -tomlkit==0.13.2 -tornado==6.4.2 -twine==6.1.0 +tomlkit==0.14.0 +tornado==6.5.5 +twine==6.2.0 typing_extensions==4.8.0 uncurl==0.0.11 -urllib3==2.3.0 -wcwidth==0.2.13 +urllib3==2.6.3 +wcwidth==0.6.0 win_unicode_console==0.5 -yt-dlp==2025.2.19 +yt-dlp==2026.3.17 zipp==3.21.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3aff3da..e1a0a91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -certifi==2025.1.31 -docutils==0.21.2 -pygments==2.19.1 -pylint==3.3.5 +certifi==2026.2.25 +docutils==0.22.4 +pygments==2.20.0 +pylint==4.0.5 pyxcute==0.8.1 -twine==6.1.0 +twine==6.2.0 diff --git a/setup.cfg b/setup.cfg index c078d79..7d96f9f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,19 +30,19 @@ packages = find: install_requires = belfrywidgets~=1.0 bidict~=0.23.1 - brotli~=1.1 - curl_cffi~=0.10.0 + brotli~=1.2 + curl_cffi~=0.14.0 deno-vm~=0.6.0 - desktop3~=0.5.3 + desktop3~=0.5.4 docopt~=0.6.2 enlighten~=1.14 - puremagic~=1.28 - pycryptodomex~=3.21 + puremagic~=2.1 + pycryptodomex~=3.23 pythreadworker~=0.10.0 safeprint~=0.2.0 uncurl~=0.0.11 - urllib3~=2.3 - yt-dlp~=2025.2 + urllib3~=2.6 + yt-dlp~=2026.3 python_requires = >=3.10 From 39d6a33e066bca1ec09c4187532fde4f7162ce39 Mon Sep 17 00:00:00 2001 From: eight04 Date: Thu, 2 Apr 2026 21:51:13 +0800 Subject: [PATCH 2/7] Fix: mime to ext --- comiccrawler/filename_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comiccrawler/filename_ext.py b/comiccrawler/filename_ext.py index a6514ab..04e51c9 100644 --- a/comiccrawler/filename_ext.py +++ b/comiccrawler/filename_ext.py @@ -6,7 +6,7 @@ import puremagic mime_dict = { - t.mime_type: t for t in puremagic.magic_header_array + t.mime_type: t for t in puremagic.main.magic_header_array } def ext_from_mime(mime): From 0c7295fcd78afba94332257d0d5429841877bb1b Mon Sep 17 00:00:00 2001 From: eight04 Date: Wed, 20 May 2026 02:22:03 +0800 Subject: [PATCH 3/7] Fix: pixiv --- comiccrawler/mods/pixiv.py | 187 +++++++++++++++++-------------------- 1 file changed, 84 insertions(+), 103 deletions(-) diff --git a/comiccrawler/mods/pixiv.py b/comiccrawler/mods/pixiv.py index 80ea220..e1b6b54 100644 --- a/comiccrawler/mods/pixiv.py +++ b/comiccrawler/mods/pixiv.py @@ -25,22 +25,6 @@ "cookie_PHPSESSID": "請輸入Cookie中的PHPSESSID" } -class DataNotFound(Exception): - pass - -def get_init_data(html): - match = re.search("""meta-global-data" content='([^']+)""", html) - if not match: - raise DataNotFound - data = json.loads(unescape(match.group(1))) - - match = re.search("""meta-preload-data" content='([^']+)""", html) - # if not match: - # raise DataNotFound - data["preload"] = json.loads(unescape(match.group(1))) - - return data - def get_title_from_init_data(html, url): init_data = get_init_data(html) user = next(iter(init_data["preload"]["user"].values())) @@ -53,22 +37,22 @@ def is_search_page(url): def get_title(html, url): if is_search_page(url): - # general title? - pass - else: - try: - return get_title_from_init_data(html, url) - except DataNotFound: - pass - return "[pixiv] " + unescape(re.search("([^<]+)", html).group(1)) + return "[pixiv] " + unescape(re.search("<title>([^<]+)", html).group(1)) + if match := re.search(r"users/(\d+)", url): + user_id = match.group(1) + title = re.search(r"og:title\" content=\"([^\"]+)", html).group(1) + return f"{user_id} - {title}" def check_login(data): if not data.get("userData"): raise PauseDownloadError("you didn't login!") def check_login_html(html): - if "pixiv.user.loggedIn = true" not in html and "login: 'yes'" not in html: - raise PauseDownloadError("you didn't login!") + if "pixiv.user.loggedIn = true" in html: + return + if re.search(r"login:\s*'yes'", html): + return + raise PauseDownloadError("you didn't login!") cache_next_page = {} @@ -77,7 +61,7 @@ def get_episodes_from_works(works): for data in sorted(works, key=lambda i: int(i["id"])): s.append(Episode( "{} - {}".format(data["id"], data["title"]), - "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={}".format(data["id"]) + "https://www.pixiv.net/artworks/{}".format(data["id"]) )) return s @@ -85,58 +69,7 @@ def get_tag_from_url(url): tags = parse_qs(urlparse(url).query).get("tag") return tags[0] if tags else None -def get_episodes_from_init_data(html, url): - init_data = get_init_data(html) - check_login(init_data) - - id = int(next(iter(init_data["preload"]["user"]))) - tag = get_tag_from_url(url) - - if tag: - def build_url(offset): - query = { - "offset": str(offset), - "limit": "48", - "tag": tag - } - return "https://www.pixiv.net/ajax/user/{}/illustmanga/tag?{}".format(id, urlencode(query)) - - response = grabhtml(build_url(0)) - response = json.loads(response) - total = response["body"]["total"] - i = 48 - pre_url = url - while i < total: - next_url = build_url(i) - cache_next_page[pre_url] = next_url - pre_url = next_url - i += 48 - return get_episodes_from_works(response["body"]["works"]) - - all = grabhtml("https://www.pixiv.net/ajax/user/{}/profile/all".format(id)) - all = json.loads(all) - - ep_ids = [int(id) for id in list(all["body"]["illusts"]) + list(all["body"]["manga"])] - ep_ids.sort() - ep_ids.reverse() - - pre_url = url - for page, i in enumerate(range(0, len(ep_ids), 48)): - ids = ep_ids[i:i + 48] - query = [("ids[]", str(id)) for id in ids] + [ - ("is_manga_top", "0"), - ("work_category", "illustManga"), - ("is_first_page", "1" if page == 0 else "0") - ] - new_url = "https://www.pixiv.net/ajax/user/{}/profile/illusts?{}".format( - id, urlencode(query)) - cache_next_page[pre_url] = new_url - pre_url = new_url - raise SkipPageError - def get_episodes_from_ajax_result(html, url): - if "ajax/user" not in url: - raise DataNotFound works = json.loads(html)["body"]["works"] if isinstance(works, dict): works = works.values() @@ -170,7 +103,47 @@ def get_episodes_from_search_ajax(html, url): cache_next_page[url] = url_o._replace(query=urlencode(query, doseq=True)).geturl() return episodes[::-1] + +def is_user_page(url): + return re.match(r"https://www\.pixiv\.net/users/\d+(/artworks)?", url) + +def get_episodes_from_user_page(html, url): + user_id = re.search(r"/users/(\d+)", url).group(1) + ajax_url = "https://www.pixiv.net/ajax/user/{}/profile/all".format(user_id) + cache_next_page[url] = ajax_url + raise SkipPageError + +def is_user_ajax_all(url): + return re.match(r"https://www\.pixiv\.net/ajax/user/\d+/profile/all", url) + +def get_episodes_from_user_ajax_all(html, url): + data = json.loads(html) + illusts = data["body"]["illusts"] + manga = data["body"]["manga"] + + ids = [int(id) for id in list(illusts) + list(manga)] + ids.sort() + ids.reverse() + + user_id = re.search(r"/user/(\d+)", url).group(1) + + pre_url = url + for page, i in enumerate(range(0, len(ids), 48)): + id_slice = ids[i:i + 48] + query = [("ids[]", str(id)) for id in id_slice] + [ + ("is_manga_top", "0"), + ("work_category", "illustManga"), + ("is_first_page", "1" if page == 0 else "0") + ] + new_url = "https://www.pixiv.net/ajax/user/{}/profile/illusts?{}".format( + user_id, urlencode(query)) + cache_next_page[pre_url] = new_url + pre_url = new_url + raise SkipPageError +def is_user_ajax_illusts(url): + return re.match(r"https://www\.pixiv\.net/ajax/user/\d+/profile/illusts", url) + def get_episodes(html, url): if is_search_page(url): return get_episodes_from_search(html, url) @@ -178,15 +151,14 @@ def get_episodes(html, url): if is_search_ajax(url): return get_episodes_from_search_ajax(html, url) - try: + if is_user_page(url): + return get_episodes_from_user_page(html, url) + + if is_user_ajax_all(url): + return get_episodes_from_user_ajax_all(html, url) + + if is_user_ajax_illusts(url): return get_episodes_from_ajax_result(html, url) - except DataNotFound: - pass - - try: - return get_episodes_from_init_data(html, url) - except DataNotFound: - pass check_login_html(html) s = [] @@ -218,23 +190,32 @@ def get_images(html, url): cache_next_page[url] = unescape(url) raise SkipPageError - init_data = get_init_data(html) - check_login(init_data) - if len(init_data["preload"]["illust"]) == 1: - illust_id, illust = init_data["preload"]["illust"].popitem() - else: - illust_id = re.search("illust_id=(\d+)", url).group(1) - illust = init_data["preload"]["illust"][illust_id] - - if illust["illustType"] != 2: # normal images - first_img = illust["urls"]["original"] - return [get_nth_img(first_img, i) for i in range(illust["pageCount"])] - - # https://www.pixiv.net/member_illust.php?mode=medium&illust_id=44298524 - ugoira_meta = "https://www.pixiv.net/ajax/illust/{}/ugoira_meta".format(illust_id) - ugoira_meta = json.loads(grabhtml(ugoira_meta)) - cache["frames"] = ugoira_meta["body"]["frames"] - return ugoira_meta["body"]["originalSrc"] + if match := re.search(r"/artworks/(\d+)", url) or re.search(r"member_illust\.php\?mode=medium&illust_id=(\d+)", url): + # https://www.pixiv.net/artworks/12345 + art_id = match.group(1) + cache_next_page[url] = f"https://www.pixiv.net/ajax/illust/{art_id}/pages" + raise SkipPageError + + if match := re.search(r"ajax/illust/(\d+)/pages", url): + data = json.loads(html) + result = [] + for p in data["body"]: + result.append(p["urls"]["original"]) + + if len(result) > 1 or "ugoira" not in result[0]: + return result + + # ugoira + # https://www.pixiv.net/member_illust.php?mode=medium&illust_id=44298524 + art_id = match.group(1) + cache_next_page[url] = f"https://www.pixiv.net/ajax/illust/{art_id}/ugoira_meta" + raise SkipPageError + + if "ugoira_meta" in url: + data = json.loads(html) + return data["body"]["originalSrc"] + + raise ValueError(f"can't find image in {url}") def errorhandler(err, crawler): if is_http(err, 404): From f5d3409dcf83da44fe7fb1e81a38449a0855659d Mon Sep 17 00:00:00 2001 From: eight04 <eight04@gmail.com> Date: Wed, 20 May 2026 02:51:15 +0800 Subject: [PATCH 4/7] install fallback? --- .github/workflows/build.yml | 45 ++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0bda6e2..2d49616 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,5 +9,48 @@ jobs: python-version: '3.10' - run: pip install -U pip wheel - run: cat requirements.txt | xargs -n 1 pip install || exit 0 - - run: pip install . + # by gemini + - name: Install Requirements with Compatibility Fallback + run: | + REQUIREMENTS_FILE="requirements-lock.txt" + + if [ ! -f "$REQUIREMENTS_FILE" ]; then + echo "Requirements file not found, skipping." + exit 0 + fi + + echo "Parsing $REQUIREMENTS_FILE..." + + while IFS= read -r line || [ -n "$line" ]; do + # Clean whitespace + line=$(echo "$line" | xargs) + + # Skip empty lines and comments + if [ -z "$line" ] || [[ "$line" =~ ^# ]]; then + continue + fi + + echo "::group::Installing $line" + + # Try exact install + if pip install "$line"; then + echo "Successfully installed exact match: $line" + else + # Exact match failed, trigger the warning and fallback + echo "::warning file=requirements.txt::Failed to install '$line' on Python ${{ matrix.python-version }}. Dropping back to latest compatible version." + + # Extract clean package name + package_name=$(echo "$line" | sed -E 's/([a-zA-Z0-9_\-]+).*/\1/') + + echo "Executing fallback: pip install $package_name" + if pip install "$package_name"; then + echo "Successfully resolved fallback for $package_name" + else + echo "::error::Both exact and fallback installations failed for $package_name" + exit 1 + fi + fi + + echo "::endgroup::" + done < "$REQUIREMENTS_FILE" - run: python cute.py test From e1f270dc5d79dd29aede21d160bfcb1200c7f332 Mon Sep 17 00:00:00 2001 From: eight04 <eight04@gmail.com> Date: Wed, 20 May 2026 02:54:47 +0800 Subject: [PATCH 5/7] pylint --- .github/workflows/build.yml | 3 ++- comiccrawler/mods/pixiv.py | 9 +-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2d49616..5523bc8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,7 +8,7 @@ jobs: with: python-version: '3.10' - run: pip install -U pip wheel - - run: cat requirements.txt | xargs -n 1 pip install || exit 0 + # - run: cat requirements.txt | xargs -n 1 pip install || exit 0 # by gemini - name: Install Requirements with Compatibility Fallback run: | @@ -53,4 +53,5 @@ jobs: echo "::endgroup::" done < "$REQUIREMENTS_FILE" + - run: pip install . - run: python cute.py test diff --git a/comiccrawler/mods/pixiv.py b/comiccrawler/mods/pixiv.py index e1b6b54..6898dc4 100644 --- a/comiccrawler/mods/pixiv.py +++ b/comiccrawler/mods/pixiv.py @@ -14,7 +14,7 @@ from urllib.parse import urljoin, urlencode, urlparse, parse_qs from zipfile import ZipFile -from ..core import Episode, grabhtml +from ..core import Episode from ..error import PauseDownloadError, is_http, SkipEpisodeError, SkipPageError from ..safeprint import print @@ -25,13 +25,6 @@ "cookie_PHPSESSID": "請輸入Cookie中的PHPSESSID" } -def get_title_from_init_data(html, url): - init_data = get_init_data(html) - user = next(iter(init_data["preload"]["user"].values())) - tag = get_tag_from_url(url) - tag = " ({})".format(tag) if tag else "" - return "{} - {}{}".format(user["userId"], user["name"], tag) - def is_search_page(url): return re.match("https://www\.pixiv\.net/tags/", url) From fe29078925c3822ae6399a5d8fc762e3f3e9a92f Mon Sep 17 00:00:00 2001 From: eight04 <eight04@gmail.com> Date: Wed, 20 May 2026 03:00:02 +0800 Subject: [PATCH 6/7] Fix: build --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5523bc8..e26fccb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,5 +53,5 @@ jobs: echo "::endgroup::" done < "$REQUIREMENTS_FILE" - - run: pip install . + # - run: pip install . - run: python cute.py test From 60188faba678e206f225a395c03009f7377dfa88 Mon Sep 17 00:00:00 2001 From: eight04 <eight04@gmail.com> Date: Wed, 20 May 2026 03:01:11 +0800 Subject: [PATCH 7/7] Change: puremagic requires python@3.12 --- .github/workflows/build.yml | 4 ++-- README.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e26fccb..0137e46 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,7 +6,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.12' - run: pip install -U pip wheel # - run: cat requirements.txt | xargs -n 1 pip install || exit 0 # by gemini @@ -53,5 +53,5 @@ jobs: echo "::endgroup::" done < "$REQUIREMENTS_FILE" - # - run: pip install . + - run: pip install . - run: python cute.py test diff --git a/README.rst b/README.rst index 6e7ead4..72bae64 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ python 後，可以直接用 pip 指令自動安裝。 Install Python ~~~~~~~~~~~~~~ -你需要 Python 3.11 以上。安裝檔可以從它的 +你需要 Python 3.12 以上。安裝檔可以從它的 `官方網站 <https://www.python.org/>`__ 下載。 安裝時記得要選「Add python.exe to path」，才能使用 pip 指令。