diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0bda6e2..0137e46 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -6,8 +6,52 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
- python-version: '3.10'
+ python-version: '3.12'
- run: pip install -U pip wheel
- - run: cat requirements.txt | xargs -n 1 pip install || exit 0
+ # - run: cat requirements.txt | xargs -n 1 pip install || exit 0
+ # by gemini
+ - name: Install Requirements with Compatibility Fallback
+ run: |
+ REQUIREMENTS_FILE="requirements-lock.txt"
+
+ if [ ! -f "$REQUIREMENTS_FILE" ]; then
+ echo "Requirements file not found, skipping."
+ exit 0
+ fi
+
+ echo "Parsing $REQUIREMENTS_FILE..."
+
+ while IFS= read -r line || [ -n "$line" ]; do
+ # Clean whitespace
+ line=$(echo "$line" | xargs)
+
+ # Skip empty lines and comments
+ if [ -z "$line" ] || [[ "$line" =~ ^# ]]; then
+ continue
+ fi
+
+ echo "::group::Installing $line"
+
+ # Try exact install
+ if pip install "$line"; then
+ echo "Successfully installed exact match: $line"
+ else
+ # Exact match failed, trigger the warning and fallback
+ echo "::warning file=requirements.txt::Failed to install '$line' on Python ${{ matrix.python-version }}. Dropping back to latest compatible version."
+
+ # Extract clean package name
+ package_name=$(echo "$line" | sed -E 's/([a-zA-Z0-9_\-]+).*/\1/')
+
+ echo "Executing fallback: pip install $package_name"
+ if pip install "$package_name"; then
+ echo "Successfully resolved fallback for $package_name"
+ else
+ echo "::error::Both exact and fallback installations failed for $package_name"
+ exit 1
+ fi
+ fi
+
+ echo "::endgroup::"
+ done < "$REQUIREMENTS_FILE"
- run: pip install .
- run: python cute.py test
diff --git a/README.rst b/README.rst
index 6e7ead4..72bae64 100644
--- a/README.rst
+++ b/README.rst
@@ -16,7 +16,7 @@ python 後,可以直接用 pip 指令自動安裝。
Install Python
~~~~~~~~~~~~~~
-你需要 Python 3.11 以上。安裝檔可以從它的
+你需要 Python 3.12 以上。安裝檔可以從它的
`官方網站 `__ 下載。
安裝時記得要選「Add python.exe to path」,才能使用 pip 指令。
diff --git a/comiccrawler/filename_ext.py b/comiccrawler/filename_ext.py
index a6514ab..04e51c9 100644
--- a/comiccrawler/filename_ext.py
+++ b/comiccrawler/filename_ext.py
@@ -6,7 +6,7 @@
import puremagic
mime_dict = {
- t.mime_type: t for t in puremagic.magic_header_array
+ t.mime_type: t for t in puremagic.main.magic_header_array
}
def ext_from_mime(mime):
diff --git a/comiccrawler/mods/pixiv.py b/comiccrawler/mods/pixiv.py
index 80ea220..6898dc4 100644
--- a/comiccrawler/mods/pixiv.py
+++ b/comiccrawler/mods/pixiv.py
@@ -14,7 +14,7 @@
from urllib.parse import urljoin, urlencode, urlparse, parse_qs
from zipfile import ZipFile
-from ..core import Episode, grabhtml
+from ..core import Episode
from ..error import PauseDownloadError, is_http, SkipEpisodeError, SkipPageError
from ..safeprint import print
@@ -25,50 +25,27 @@
"cookie_PHPSESSID": "請輸入Cookie中的PHPSESSID"
}
-class DataNotFound(Exception):
- pass
-
-def get_init_data(html):
- match = re.search("""meta-global-data" content='([^']+)""", html)
- if not match:
- raise DataNotFound
- data = json.loads(unescape(match.group(1)))
-
- match = re.search("""meta-preload-data" content='([^']+)""", html)
- # if not match:
- # raise DataNotFound
- data["preload"] = json.loads(unescape(match.group(1)))
-
- return data
-
-def get_title_from_init_data(html, url):
- init_data = get_init_data(html)
- user = next(iter(init_data["preload"]["user"].values()))
- tag = get_tag_from_url(url)
- tag = " ({})".format(tag) if tag else ""
- return "{} - {}{}".format(user["userId"], user["name"], tag)
-
def is_search_page(url):
return re.match("https://www\.pixiv\.net/tags/", url)
def get_title(html, url):
if is_search_page(url):
- # general title?
- pass
- else:
- try:
- return get_title_from_init_data(html, url)
- except DataNotFound:
- pass
- return "[pixiv] " + unescape(re.search("
([^<]+)", html).group(1))
+ return "[pixiv] " + unescape(re.search("([^<]+)", html).group(1))
+ if match := re.search(r"users/(\d+)", url):
+ user_id = match.group(1)
+ title = re.search(r"og:title\" content=\"([^\"]+)", html).group(1)
+ return f"{user_id} - {title}"
def check_login(data):
if not data.get("userData"):
raise PauseDownloadError("you didn't login!")
def check_login_html(html):
- if "pixiv.user.loggedIn = true" not in html and "login: 'yes'" not in html:
- raise PauseDownloadError("you didn't login!")
+ if "pixiv.user.loggedIn = true" in html:
+ return
+ if re.search(r"login:\s*'yes'", html):
+ return
+ raise PauseDownloadError("you didn't login!")
cache_next_page = {}
@@ -77,7 +54,7 @@ def get_episodes_from_works(works):
for data in sorted(works, key=lambda i: int(i["id"])):
s.append(Episode(
"{} - {}".format(data["id"], data["title"]),
- "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={}".format(data["id"])
+ "https://www.pixiv.net/artworks/{}".format(data["id"])
))
return s
@@ -85,58 +62,7 @@ def get_tag_from_url(url):
tags = parse_qs(urlparse(url).query).get("tag")
return tags[0] if tags else None
-def get_episodes_from_init_data(html, url):
- init_data = get_init_data(html)
- check_login(init_data)
-
- id = int(next(iter(init_data["preload"]["user"])))
- tag = get_tag_from_url(url)
-
- if tag:
- def build_url(offset):
- query = {
- "offset": str(offset),
- "limit": "48",
- "tag": tag
- }
- return "https://www.pixiv.net/ajax/user/{}/illustmanga/tag?{}".format(id, urlencode(query))
-
- response = grabhtml(build_url(0))
- response = json.loads(response)
- total = response["body"]["total"]
- i = 48
- pre_url = url
- while i < total:
- next_url = build_url(i)
- cache_next_page[pre_url] = next_url
- pre_url = next_url
- i += 48
- return get_episodes_from_works(response["body"]["works"])
-
- all = grabhtml("https://www.pixiv.net/ajax/user/{}/profile/all".format(id))
- all = json.loads(all)
-
- ep_ids = [int(id) for id in list(all["body"]["illusts"]) + list(all["body"]["manga"])]
- ep_ids.sort()
- ep_ids.reverse()
-
- pre_url = url
- for page, i in enumerate(range(0, len(ep_ids), 48)):
- ids = ep_ids[i:i + 48]
- query = [("ids[]", str(id)) for id in ids] + [
- ("is_manga_top", "0"),
- ("work_category", "illustManga"),
- ("is_first_page", "1" if page == 0 else "0")
- ]
- new_url = "https://www.pixiv.net/ajax/user/{}/profile/illusts?{}".format(
- id, urlencode(query))
- cache_next_page[pre_url] = new_url
- pre_url = new_url
- raise SkipPageError
-
def get_episodes_from_ajax_result(html, url):
- if "ajax/user" not in url:
- raise DataNotFound
works = json.loads(html)["body"]["works"]
if isinstance(works, dict):
works = works.values()
@@ -170,7 +96,47 @@ def get_episodes_from_search_ajax(html, url):
cache_next_page[url] = url_o._replace(query=urlencode(query, doseq=True)).geturl()
return episodes[::-1]
+
+def is_user_page(url):
+ return re.match(r"https://www\.pixiv\.net/users/\d+(/artworks)?", url)
+
+def get_episodes_from_user_page(html, url):
+ user_id = re.search(r"/users/(\d+)", url).group(1)
+ ajax_url = "https://www.pixiv.net/ajax/user/{}/profile/all".format(user_id)
+ cache_next_page[url] = ajax_url
+ raise SkipPageError
+
+def is_user_ajax_all(url):
+ return re.match(r"https://www\.pixiv\.net/ajax/user/\d+/profile/all", url)
+
+def get_episodes_from_user_ajax_all(html, url):
+ data = json.loads(html)
+ illusts = data["body"]["illusts"]
+ manga = data["body"]["manga"]
+
+ ids = [int(id) for id in list(illusts) + list(manga)]
+ ids.sort()
+ ids.reverse()
+
+ user_id = re.search(r"/user/(\d+)", url).group(1)
+
+ pre_url = url
+ for page, i in enumerate(range(0, len(ids), 48)):
+ id_slice = ids[i:i + 48]
+ query = [("ids[]", str(id)) for id in id_slice] + [
+ ("is_manga_top", "0"),
+ ("work_category", "illustManga"),
+ ("is_first_page", "1" if page == 0 else "0")
+ ]
+ new_url = "https://www.pixiv.net/ajax/user/{}/profile/illusts?{}".format(
+ user_id, urlencode(query))
+ cache_next_page[pre_url] = new_url
+ pre_url = new_url
+ raise SkipPageError
+def is_user_ajax_illusts(url):
+ return re.match(r"https://www\.pixiv\.net/ajax/user/\d+/profile/illusts", url)
+
def get_episodes(html, url):
if is_search_page(url):
return get_episodes_from_search(html, url)
@@ -178,15 +144,14 @@ def get_episodes(html, url):
if is_search_ajax(url):
return get_episodes_from_search_ajax(html, url)
- try:
+ if is_user_page(url):
+ return get_episodes_from_user_page(html, url)
+
+ if is_user_ajax_all(url):
+ return get_episodes_from_user_ajax_all(html, url)
+
+ if is_user_ajax_illusts(url):
return get_episodes_from_ajax_result(html, url)
- except DataNotFound:
- pass
-
- try:
- return get_episodes_from_init_data(html, url)
- except DataNotFound:
- pass
check_login_html(html)
s = []
@@ -218,23 +183,32 @@ def get_images(html, url):
cache_next_page[url] = unescape(url)
raise SkipPageError
- init_data = get_init_data(html)
- check_login(init_data)
- if len(init_data["preload"]["illust"]) == 1:
- illust_id, illust = init_data["preload"]["illust"].popitem()
- else:
- illust_id = re.search("illust_id=(\d+)", url).group(1)
- illust = init_data["preload"]["illust"][illust_id]
-
- if illust["illustType"] != 2: # normal images
- first_img = illust["urls"]["original"]
- return [get_nth_img(first_img, i) for i in range(illust["pageCount"])]
-
- # https://www.pixiv.net/member_illust.php?mode=medium&illust_id=44298524
- ugoira_meta = "https://www.pixiv.net/ajax/illust/{}/ugoira_meta".format(illust_id)
- ugoira_meta = json.loads(grabhtml(ugoira_meta))
- cache["frames"] = ugoira_meta["body"]["frames"]
- return ugoira_meta["body"]["originalSrc"]
+ if match := re.search(r"/artworks/(\d+)", url) or re.search(r"member_illust\.php\?mode=medium&illust_id=(\d+)", url):
+ # https://www.pixiv.net/artworks/12345
+ art_id = match.group(1)
+ cache_next_page[url] = f"https://www.pixiv.net/ajax/illust/{art_id}/pages"
+ raise SkipPageError
+
+ if match := re.search(r"ajax/illust/(\d+)/pages", url):
+ data = json.loads(html)
+ result = []
+ for p in data["body"]:
+ result.append(p["urls"]["original"])
+
+ if len(result) > 1 or "ugoira" not in result[0]:
+ return result
+
+ # ugoira
+ # https://www.pixiv.net/member_illust.php?mode=medium&illust_id=44298524
+ art_id = match.group(1)
+ cache_next_page[url] = f"https://www.pixiv.net/ajax/illust/{art_id}/ugoira_meta"
+ raise SkipPageError
+
+ if "ugoira_meta" in url:
+ data = json.loads(html)
+ return data["body"]["originalSrc"]
+
+ raise ValueError(f"can't find image in {url}")
def errorhandler(err, crawler):
if is_http(err, 404):
diff --git a/requirements-lock.txt b/requirements-lock.txt
index de4dd1b..82e4785 100644
--- a/requirements-lock.txt
+++ b/requirements-lock.txt
@@ -1,66 +1,66 @@
ansicon==1.89.0
-astroid==3.3.9
+astroid==4.0.4
belfrywidgets==1.0.3
bidict==0.23.1
-blessed==1.20.0
-Brotli==1.1.0
-certifi==2025.1.31
-cffi==1.17.1
-charset-normalizer==3.4.1
+blessed==1.38.0
+brotli==1.2.0
+certifi==2026.2.25
+cffi==2.0.0
+charset-normalizer==3.4.7
colorama==0.4.6
-curl_cffi==0.10.0
+curl_cffi==0.14.0
deno_vm==0.6.0
-desktop3==0.5.3
-dill==0.3.9
+desktop3==0.5.4
+dill==0.4.1
docopt==0.6.2
-docutils==0.21.2
+docutils==0.22.4
enlighten==1.14.1
-id==1.5.0
-idna==3.10
+id==1.6.1
+idna==3.11
importlib_metadata==8.5.0
-isort==6.0.1
+isort==8.0.1
jaraco.classes==3.4.0
-jaraco.context==6.0.1
-jaraco.functools==4.1.0
-jinxed==1.3.0
-keyring==25.6.0
+jaraco.context==6.1.2
+jaraco.functools==4.4.0
+jinxed==1.4.0
+keyring==25.7.0
livereload==2.7.1
-markdown-it-py==3.0.0
+markdown-it-py==4.0.0
mccabe==0.7.0
mdurl==0.1.2
-more-itertools==10.6.0
-nh3==0.2.21
+more-itertools==10.8.0
+nh3==0.3.4
ordered-set==3.1.1
-packaging==24.2
+packaging==26.0
pkginfo==1.10.0
-platformdirs==4.3.6
+platformdirs==4.9.4
prefixed==0.9.0
-puremagic==1.28
-pycparser==2.22
-pycryptodomex==3.21.0
-Pygments==2.19.1
-pylint==3.3.5
-pyperclip==1.9.0
+puremagic==2.1.1
+pycparser==3.0
+pycryptodomex==3.23.0
+Pygments==2.20.0
+pylint==4.0.5
+pyperclip==1.11.0
pythreadworker==0.10.0
pywin32-ctypes==0.2.3
pyxcute==0.8.1
readme_renderer==44.0
-requests==2.32.3
+requests==2.33.1
requests-toolbelt==1.0.0
rfc3986==2.0.0
-rich==13.9.4
+rich==14.3.3
safeprint==0.2.0
semver==2.13.0
Send2Trash==1.8.3
-setuptools==76.0.0
+setuptools==82.0.1
six==1.17.0
-tomlkit==0.13.2
-tornado==6.4.2
-twine==6.1.0
+tomlkit==0.14.0
+tornado==6.5.5
+twine==6.2.0
typing_extensions==4.8.0
uncurl==0.0.11
-urllib3==2.3.0
-wcwidth==0.2.13
+urllib3==2.6.3
+wcwidth==0.6.0
win_unicode_console==0.5
-yt-dlp==2025.2.19
+yt-dlp==2026.3.17
zipp==3.21.0
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 3aff3da..e1a0a91 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
-certifi==2025.1.31
-docutils==0.21.2
-pygments==2.19.1
-pylint==3.3.5
+certifi==2026.2.25
+docutils==0.22.4
+pygments==2.20.0
+pylint==4.0.5
pyxcute==0.8.1
-twine==6.1.0
+twine==6.2.0
diff --git a/setup.cfg b/setup.cfg
index c078d79..7d96f9f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -30,19 +30,19 @@ packages = find:
install_requires =
belfrywidgets~=1.0
bidict~=0.23.1
- brotli~=1.1
- curl_cffi~=0.10.0
+ brotli~=1.2
+ curl_cffi~=0.14.0
deno-vm~=0.6.0
- desktop3~=0.5.3
+ desktop3~=0.5.4
docopt~=0.6.2
enlighten~=1.14
- puremagic~=1.28
- pycryptodomex~=3.21
+ puremagic~=2.1
+ pycryptodomex~=3.23
pythreadworker~=0.10.0
safeprint~=0.2.0
uncurl~=0.0.11
- urllib3~=2.3
- yt-dlp~=2025.2
+ urllib3~=2.6
+ yt-dlp~=2026.3
python_requires = >=3.10