diff --git a/CONFIG.md b/CONFIG.md index b87f75c..63a7b01 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -92,6 +92,20 @@ common to all crawlers: load for the crawl target. (Default: `0.0`) - `windows_paths`: Whether PFERD should find alternative names for paths that are invalid on Windows. (Default: `yes` on Windows, `no` otherwise) +- `unicode_normalization`: Which [Unicode normalization form][unicode-norm] to + apply to file and directory names before they are written to disk. This is + useful on macOS/iOS, where AirDrop silently drops files whose path crosses a + precomposed (NFC) non-ASCII directory name; normalizing to `nfd` avoids this. + (Default: `none`) + - `none`: Names are written as-is, in whatever form the crawler produced + (usually NFC). + - `nfc`, `nfd`, `nfkc`, `nfkd`: Names are normalized to the respective form. + + Note: changing this option on an already-downloaded directory may cause a + one-time re-download and cleanup, since the on-disk names and the previous + report use the old form. Starting from a fresh `output_dir` avoids the churn. + +[unicode-norm]: "unicodedata.normalize" Some crawlers may also require credentials for authentication. To configure how the crawler obtains its credentials, the `auth` option is used. It is set to the diff --git a/LICENSE b/LICENSE index ccccbe3..97b50aa 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ Copyright 2019-2024 Garmelon, I-Al-Istannen, danstooamerican, pavelzw, TheChristophe, Scriptim, thelukasprobst, Toorero, - Mr-Pine, p-fruck, PinieP + Mr-Pine, p-fruck, PinieP, Florian Raith Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/PFERD/crawl/crawler.py b/PFERD/crawl/crawler.py index e2cdf30..ec4d7f0 100644 --- a/PFERD/crawl/crawler.py +++ b/PFERD/crawl/crawler.py @@ -11,7 +11,15 @@ from ..deduplicator import Deduplicator from ..limiter import Limiter from ..logging import ProgressBar, log -from ..output_dir import FileSink, FileSinkToken, OnConflict, OutputDirectory, OutputDirError, Redownload +from ..output_dir import ( + FileSink, + FileSinkToken, + OnConflict, + OutputDirectory, + OutputDirError, + Redownload, + UnicodeNormalization, +) from ..report import MarkConflictError, MarkDuplicateError, Report from ..transformer import Transformer from ..utils import ReusableAsyncContextManager, fmt_path @@ -175,6 +183,17 @@ def on_conflict(self) -> OnConflict: str(e).capitalize(), ) + def unicode_normalization(self) -> UnicodeNormalization: + value = self.s.get("unicode_normalization", "none") + try: + return UnicodeNormalization.from_string(value) + except ValueError as e: + self.invalid_value( + "unicode_normalization", + value, + str(e).capitalize(), + ) + def transform(self) -> str: return self.s.get("transform", "") @@ -247,6 +266,7 @@ def __init__( config.default_section.working_dir() / section.output_dir(name), section.redownload(), section.on_conflict(), + section.unicode_normalization(), ) @property diff --git a/PFERD/output_dir.py b/PFERD/output_dir.py index 159e1db..fea69cf 100644 --- a/PFERD/output_dir.py +++ b/PFERD/output_dir.py @@ -4,6 +4,7 @@ import random import shutil import string +import unicodedata from collections.abc import Iterator from contextlib import contextmanager, suppress from dataclasses import dataclass @@ -57,6 +58,27 @@ def from_string(string: str) -> "OnConflict": ) from None +class UnicodeNormalization(Enum): + NONE = "none" + NFC = "nfc" + NFD = "nfd" + NFKC = "nfkc" + NFKD = "nfkd" + + @staticmethod + def from_string(string: str) -> "UnicodeNormalization": + try: + return UnicodeNormalization(string) + except ValueError: + raise ValueError("must be one of 'none', 'nfc', 'nfd', 'nfkc', 'nfkd'") from None + + def normalize(self, path: PurePath) -> PurePath: + if self == UnicodeNormalization.NONE: + return path + form = self.name # "NFC", "NFD", "NFKC" or "NFKD" + return PurePath(*(unicodedata.normalize(form, part) for part in path.parts)) + + @dataclass class Heuristics: etag_differs: Optional[bool] @@ -146,6 +168,7 @@ def __init__( root: Path, redownload: Redownload, on_conflict: OnConflict, + unicode_normalization: UnicodeNormalization = UnicodeNormalization.NONE, ): if os.name == "nt": # Windows limits the path length to 260 for some historical reason. @@ -158,6 +181,7 @@ def __init__( self._redownload = redownload self._on_conflict = on_conflict + self._unicode_normalization = unicode_normalization self._report_path = self.resolve(self.REPORT_FILE) self._report = Report() @@ -384,6 +408,7 @@ def should_try_download( redownload: Optional[Redownload] = None, on_conflict: Optional[OnConflict] = None, ) -> bool: + path = self._unicode_normalization.normalize(path) heuristics = Heuristics(etag_differs, mtime) redownload = self._redownload if redownload is None else redownload on_conflict = self._on_conflict if on_conflict is None else on_conflict @@ -406,6 +431,7 @@ async def download( MarkConflictError. """ + path = self._unicode_normalization.normalize(path) heuristics = Heuristics(etag_differs, mtime) redownload = self._redownload if redownload is None else redownload on_conflict = self._on_conflict if on_conflict is None else on_conflict