Fix: pixiv, bump deps

.github/workflows/build.yml

@@ -6,8 +6,52 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.10'
+          python-version: '3.12'
       - run: pip install -U pip wheel
-      - run: cat requirements.txt | xargs -n 1 pip install || exit 0
+      # - run: cat requirements.txt | xargs -n 1 pip install || exit 0
+      # by gemini
+      - name: Install Requirements with Compatibility Fallback
+        run: |
+          REQUIREMENTS_FILE="requirements-lock.txt"
+
+          if [ ! -f "$REQUIREMENTS_FILE" ]; then
+            echo "Requirements file not found, skipping."
+            exit 0
+          fi
+
+          echo "Parsing $REQUIREMENTS_FILE..."
+
+          while IFS= read -r line || [ -n "$line" ]; do
+            # Clean whitespace
+            line=$(echo "$line" | xargs)
+
+            # Skip empty lines and comments
+            if [ -z "$line" ] || [[ "$line" =~ ^# ]]; then
+              continue
+            fi
+
+            echo "::group::Installing $line"
+
+            # Try exact install
+            if pip install "$line"; then
+              echo "Successfully installed exact match: $line"
+            else
+              # Exact match failed, trigger the warning and fallback
+              echo "::warning file=requirements.txt::Failed to install '$line' on Python ${{ matrix.python-version }}. Dropping back to latest compatible version."
+
+              # Extract clean package name
+              package_name=$(echo "$line" | sed -E 's/([a-zA-Z0-9_\-]+).*/\1/')
+
+              echo "Executing fallback: pip install $package_name"
+              if pip install "$package_name"; then
+                echo "Successfully resolved fallback for $package_name"
+              else
+                echo "::error::Both exact and fallback installations failed for $package_name"
+                exit 1
+              fi
+            fi
+
+            echo "::endgroup::"
+          done < "$REQUIREMENTS_FILE"
       - run: pip install .
       - run: python cute.py test

README.rst

@@ -16,7 +16,7 @@ python 後，可以直接用 pip 指令自動安裝。
 Install Python
 ~~~~~~~~~~~~~~
 
-你需要 Python 3.11 以上。安裝檔可以從它的
+你需要 Python 3.12 以上。安裝檔可以從它的
 `官方網站 <https://www.python.org/>`__ 下載。
 
 安裝時記得要選「Add python.exe to path」，才能使用 pip 指令。

comiccrawler/filename_ext.py

@@ -6,7 +6,7 @@
 import puremagic
 
 mime_dict = {
-	t.mime_type: t for t in puremagic.magic_header_array
+	t.mime_type: t for t in puremagic.main.magic_header_array
 	}
 
 def ext_from_mime(mime):

comiccrawler/mods/pixiv.py

@@ -14,7 +14,7 @@
 from urllib.parse import urljoin, urlencode, urlparse, parse_qs
 from zipfile import ZipFile
 
-from ..core import Episode, grabhtml
+from ..core import Episode
 from ..error import PauseDownloadError, is_http, SkipEpisodeError, SkipPageError
 from ..safeprint import print
 
@@ -25,50 +25,27 @@
 	"cookie_PHPSESSID": "請輸入Cookie中的PHPSESSID"
 }
 
-class DataNotFound(Exception):
-	pass
-
-def get_init_data(html):
-	match = re.search("""meta-global-data" content='([^']+)""", html)
-	if not match:
-		raise DataNotFound
-	data = json.loads(unescape(match.group(1)))
-
-	match = re.search("""meta-preload-data" content='([^']+)""", html)
-	# if not match:
-		# raise DataNotFound
-	data["preload"] = json.loads(unescape(match.group(1)))
-
-	return data
-
-def get_title_from_init_data(html, url):
-	init_data = get_init_data(html)
-	user = next(iter(init_data["preload"]["user"].values()))
-	tag = get_tag_from_url(url)
-	tag = " ({})".format(tag) if tag else ""
-	return "{} - {}{}".format(user["userId"], user["name"], tag)
-
 def is_search_page(url):
 	return re.match("https://www\.pixiv\.net/tags/", url)
 
 def get_title(html, url):
 	if is_search_page(url):
-		# general title?
-		pass
-	else:
-		try:
-			return get_title_from_init_data(html, url)
-		except DataNotFound:
-			pass
-	return "[pixiv] " + unescape(re.search("<title>([^<]+)", html).group(1))
+		return "[pixiv] " + unescape(re.search("<title>([^<]+)", html).group(1))
+	if match := re.search(r"users/(\d+)", url):
+		user_id = match.group(1)
+		title = re.search(r"og:title\" content=\"([^\"]+)", html).group(1)
+		return f"{user_id} - {title}"
 
 def check_login(data):
 	if not data.get("userData"):
 		raise PauseDownloadError("you didn't login!")
 
 def check_login_html(html):
-	if "pixiv.user.loggedIn = true" not in html and "login: 'yes'" not in html:
-		raise PauseDownloadError("you didn't login!")
+	if "pixiv.user.loggedIn = true" in html:
+		return
+	if re.search(r"login:\s*'yes'", html):
+		return
+	raise PauseDownloadError("you didn't login!")
 
 cache_next_page = {}
 
@@ -77,66 +54,15 @@ def get_episodes_from_works(works):
 	for data in sorted(works, key=lambda i: int(i["id"])):
 		s.append(Episode(
 			"{} - {}".format(data["id"], data["title"]),
-			"https://www.pixiv.net/member_illust.php?mode=medium&illust_id={}".format(data["id"])
+			"https://www.pixiv.net/artworks/{}".format(data["id"])
 		))
 	return s
 
 def get_tag_from_url(url):
 	tags = parse_qs(urlparse(url).query).get("tag")
 	return tags[0] if tags else None
 
-def get_episodes_from_init_data(html, url):
-	init_data = get_init_data(html)
-	check_login(init_data)
-
-	id = int(next(iter(init_data["preload"]["user"])))
-	tag = get_tag_from_url(url)
-
-	if tag:
-		def build_url(offset):
-			query = {
-				"offset": str(offset),
-				"limit": "48",
-				"tag": tag
-			}
-			return "https://www.pixiv.net/ajax/user/{}/illustmanga/tag?{}".format(id, urlencode(query))
-
-		response = grabhtml(build_url(0))
-		response = json.loads(response)
-		total = response["body"]["total"]
-		i = 48
-		pre_url = url
-		while i < total:
-			next_url = build_url(i)
-			cache_next_page[pre_url] = next_url
-			pre_url = next_url
-			i += 48
-		return get_episodes_from_works(response["body"]["works"])
-
-	all = grabhtml("https://www.pixiv.net/ajax/user/{}/profile/all".format(id))
-	all = json.loads(all)
-
-	ep_ids = [int(id) for id in list(all["body"]["illusts"]) + list(all["body"]["manga"])]
-	ep_ids.sort()
-	ep_ids.reverse()
-
-	pre_url = url
-	for page, i in enumerate(range(0, len(ep_ids), 48)):
-		ids = ep_ids[i:i + 48]
-		query = [("ids[]", str(id)) for id in ids] + [
-			("is_manga_top", "0"),
-			("work_category", "illustManga"),
-			("is_first_page", "1" if page == 0 else "0")
-		]
-		new_url = "https://www.pixiv.net/ajax/user/{}/profile/illusts?{}".format(
-			id, urlencode(query))
-		cache_next_page[pre_url] = new_url
-		pre_url = new_url
-	raise SkipPageError
-
 def get_episodes_from_ajax_result(html, url):
-	if "ajax/user" not in url:
-		raise DataNotFound
 	works = json.loads(html)["body"]["works"]
 	if isinstance(works, dict):
 		works = works.values()
@@ -170,23 +96,62 @@ def get_episodes_from_search_ajax(html, url):
 		cache_next_page[url] = url_o._replace(query=urlencode(query, doseq=True)).geturl()
 
 	return episodes[::-1]
+
+def is_user_page(url):
+	return re.match(r"https://www\.pixiv\.net/users/\d+(/artworks)?", url)
+
+def get_episodes_from_user_page(html, url):
+	user_id = re.search(r"/users/(\d+)", url).group(1)
+	ajax_url = "https://www.pixiv.net/ajax/user/{}/profile/all".format(user_id)
+	cache_next_page[url] = ajax_url
+	raise SkipPageError
+
+def is_user_ajax_all(url):
+	return re.match(r"https://www\.pixiv\.net/ajax/user/\d+/profile/all", url)
+
+def get_episodes_from_user_ajax_all(html, url):
+	data = json.loads(html)
+	illusts = data["body"]["illusts"]
+	manga = data["body"]["manga"]
+
+	ids = [int(id) for id in list(illusts) + list(manga)]
+	ids.sort()
+	ids.reverse()
+
+	user_id = re.search(r"/user/(\d+)", url).group(1)
+
+	pre_url = url
+	for page, i in enumerate(range(0, len(ids), 48)):
+		id_slice = ids[i:i + 48]
+		query = [("ids[]", str(id)) for id in id_slice] + [
+			("is_manga_top", "0"),
+			("work_category", "illustManga"),
+			("is_first_page", "1" if page == 0 else "0")
+		]
+		new_url = "https://www.pixiv.net/ajax/user/{}/profile/illusts?{}".format(
+			user_id, urlencode(query))
+		cache_next_page[pre_url] = new_url
+		pre_url = new_url
+	raise SkipPageError
 
+def is_user_ajax_illusts(url):
+	return re.match(r"https://www\.pixiv\.net/ajax/user/\d+/profile/illusts", url)
+
 def get_episodes(html, url):
 	if is_search_page(url):
 		return get_episodes_from_search(html, url)
 
 	if is_search_ajax(url):
 		return get_episodes_from_search_ajax(html, url)
 
-	try:
+	if is_user_page(url):
+		return get_episodes_from_user_page(html, url)
+
+	if is_user_ajax_all(url):
+		return get_episodes_from_user_ajax_all(html, url)
+
+	if is_user_ajax_illusts(url):
 		return get_episodes_from_ajax_result(html, url)
-	except DataNotFound:
-		pass
-
-	try:
-		return get_episodes_from_init_data(html, url)
-	except DataNotFound:
-		pass
 
 	check_login_html(html)
 	s = []
@@ -218,23 +183,32 @@ def get_images(html, url):
 		cache_next_page[url] = unescape(url)
 		raise SkipPageError
 
-	init_data = get_init_data(html)
-	check_login(init_data)
-	if len(init_data["preload"]["illust"]) == 1:
-		illust_id, illust = init_data["preload"]["illust"].popitem()
-	else:
-		illust_id = re.search("illust_id=(\d+)", url).group(1)
-		illust = init_data["preload"]["illust"][illust_id]
-
-	if illust["illustType"] != 2: # normal images
-		first_img = illust["urls"]["original"]
-		return [get_nth_img(first_img, i) for i in range(illust["pageCount"])]
-
-	# https://www.pixiv.net/member_illust.php?mode=medium&illust_id=44298524
-	ugoira_meta = "https://www.pixiv.net/ajax/illust/{}/ugoira_meta".format(illust_id)
-	ugoira_meta = json.loads(grabhtml(ugoira_meta))
-	cache["frames"] = ugoira_meta["body"]["frames"]
-	return ugoira_meta["body"]["originalSrc"]
+	if match := re.search(r"/artworks/(\d+)", url) or re.search(r"member_illust\.php\?mode=medium&illust_id=(\d+)", url):
+		# https://www.pixiv.net/artworks/12345
+		art_id = match.group(1)
+		cache_next_page[url] = f"https://www.pixiv.net/ajax/illust/{art_id}/pages"
+		raise SkipPageError
+
+	if match := re.search(r"ajax/illust/(\d+)/pages", url):
+		data = json.loads(html)
+		result = []
+		for p in data["body"]:
+			result.append(p["urls"]["original"])
+
+		if len(result) > 1 or "ugoira" not in result[0]:
+			return result
+
+		# ugoira
+		# https://www.pixiv.net/member_illust.php?mode=medium&illust_id=44298524
+		art_id = match.group(1)
+		cache_next_page[url] = f"https://www.pixiv.net/ajax/illust/{art_id}/ugoira_meta"
+		raise SkipPageError
+
+	if "ugoira_meta" in url:
+		data = json.loads(html)
+		return data["body"]["originalSrc"]
+
+	raise ValueError(f"can't find image in {url}")
 
 def errorhandler(err, crawler):
 	if is_http(err, 404):