Sanitize Unicode normalization form for file and directory names

CONFIG.md

@@ -92,6 +92,20 @@ common to all crawlers:
   load for the crawl target. (Default: `0.0`)
 - `windows_paths`: Whether PFERD should find alternative names for paths that
   are invalid on Windows. (Default: `yes` on Windows, `no` otherwise)
+- `unicode_normalization`: Which [Unicode normalization form][unicode-norm] to
+  apply to file and directory names before they are written to disk. This is
+  useful on macOS/iOS, where AirDrop silently drops files whose path crosses a
+  precomposed (NFC) non-ASCII directory name; normalizing to `nfd` avoids this.
+  (Default: `none`)
+    - `none`: Names are written as-is, in whatever form the crawler produced
+      (usually NFC).
+    - `nfc`, `nfd`, `nfkc`, `nfkd`: Names are normalized to the respective form.
+
+  Note: changing this option on an already-downloaded directory may cause a
+  one-time re-download and cleanup, since the on-disk names and the previous
+  report use the old form. Starting from a fresh `output_dir` avoids the churn.
+
+[unicode-norm]: <https://docs.python.org/3/library/unicodedata.html#unicodedata.normalize> "unicodedata.normalize"
 
 Some crawlers may also require credentials for authentication. To configure how
 the crawler obtains its credentials, the `auth` option is used. It is set to the

LICENSE

@@ -1,6 +1,6 @@
 Copyright 2019-2024 Garmelon, I-Al-Istannen, danstooamerican, pavelzw,
                     TheChristophe, Scriptim, thelukasprobst, Toorero,
-                    Mr-Pine, p-fruck, PinieP
+                    Mr-Pine, p-fruck, PinieP, Florian Raith
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

PFERD/crawl/crawler.py

@@ -11,7 +11,15 @@
 from ..deduplicator import Deduplicator
 from ..limiter import Limiter
 from ..logging import ProgressBar, log
-from ..output_dir import FileSink, FileSinkToken, OnConflict, OutputDirectory, OutputDirError, Redownload
+from ..output_dir import (
+    FileSink,
+    FileSinkToken,
+    OnConflict,
+    OutputDirectory,
+    OutputDirError,
+    Redownload,
+    UnicodeNormalization,
+)
 from ..report import MarkConflictError, MarkDuplicateError, Report
 from ..transformer import Transformer
 from ..utils import ReusableAsyncContextManager, fmt_path
@@ -175,6 +183,17 @@ def on_conflict(self) -> OnConflict:
                 str(e).capitalize(),
             )
 
+    def unicode_normalization(self) -> UnicodeNormalization:
+        value = self.s.get("unicode_normalization", "none")
+        try:
+            return UnicodeNormalization.from_string(value)
+        except ValueError as e:
+            self.invalid_value(
+                "unicode_normalization",
+                value,
+                str(e).capitalize(),
+            )
+
     def transform(self) -> str:
         return self.s.get("transform", "")
 
@@ -247,6 +266,7 @@ def __init__(
             config.default_section.working_dir() / section.output_dir(name),
             section.redownload(),
             section.on_conflict(),
+            section.unicode_normalization(),
         )
 
     @property

PFERD/output_dir.py

@@ -4,6 +4,7 @@
 import random
 import shutil
 import string
+import unicodedata
 from collections.abc import Iterator
 from contextlib import contextmanager, suppress
 from dataclasses import dataclass
@@ -57,6 +58,27 @@ def from_string(string: str) -> "OnConflict":
             ) from None
 
 
+class UnicodeNormalization(Enum):
+    NONE = "none"
+    NFC = "nfc"
+    NFD = "nfd"
+    NFKC = "nfkc"
+    NFKD = "nfkd"
+
+    @staticmethod
+    def from_string(string: str) -> "UnicodeNormalization":
+        try:
+            return UnicodeNormalization(string)
+        except ValueError:
+            raise ValueError("must be one of 'none', 'nfc', 'nfd', 'nfkc', 'nfkd'") from None
+
+    def normalize(self, path: PurePath) -> PurePath:
+        if self == UnicodeNormalization.NONE:
+            return path
+        form = self.name  # "NFC", "NFD", "NFKC" or "NFKD"
+        return PurePath(*(unicodedata.normalize(form, part) for part in path.parts))
+
+
 @dataclass
 class Heuristics:
     etag_differs: Optional[bool]
@@ -146,6 +168,7 @@ def __init__(
         root: Path,
         redownload: Redownload,
         on_conflict: OnConflict,
+        unicode_normalization: UnicodeNormalization = UnicodeNormalization.NONE,
     ):
         if os.name == "nt":
             # Windows limits the path length to 260 for some historical reason.
@@ -158,6 +181,7 @@ def __init__(
 
         self._redownload = redownload
         self._on_conflict = on_conflict
+        self._unicode_normalization = unicode_normalization
 
         self._report_path = self.resolve(self.REPORT_FILE)
         self._report = Report()
@@ -384,6 +408,7 @@ def should_try_download(
         redownload: Optional[Redownload] = None,
         on_conflict: Optional[OnConflict] = None,
     ) -> bool:
+        path = self._unicode_normalization.normalize(path)
         heuristics = Heuristics(etag_differs, mtime)
         redownload = self._redownload if redownload is None else redownload
         on_conflict = self._on_conflict if on_conflict is None else on_conflict
@@ -406,6 +431,7 @@ async def download(
         MarkConflictError.
         """
 
+        path = self._unicode_normalization.normalize(path)
         heuristics = Heuristics(etag_differs, mtime)
         redownload = self._redownload if redownload is None else redownload
         on_conflict = self._on_conflict if on_conflict is None else on_conflict