From f9de6658e732977faa69da7f3775e53e2e176446 Mon Sep 17 00:00:00 2001 From: Niaz Date: Sun, 21 Jun 2026 10:57:16 +0200 Subject: [PATCH] backend: add Windows-native dev/test setup The backend normally runs in the Linux container. This adds a reproducible way to install the full pinned dependency set and run the test suite natively on a Windows dev box (Python 3.11, in-memory SQLite) -- verified end to end at 1330/1363 tests passing (remaining failures are environmental: Windows case-insensitive FS, path separators, timestamp-tie ordering, exiftool/mock test scaffolding -- not install/import failures). Native path (primary): - requirements.windows.txt: reuses every pin via -r requirements.txt / requirements.dev.txt and only swaps the Windows-friendly variants (pyvips[binary], python-magic-bin, psycopg[binary]) plus the llama-cpp-python CPU wheel index. - scripts/setup_windows.ps1: venv + pip install + winget exiftool/ImageMagick + a venv libmagic loader shim (python-magic otherwise loads Git-for-Windows' MSYS libmagic and segfaults CPython). - scripts/run_tests_windows.ps1: runs the suite against test_sqlite. Lightweight fallback (no native deps / no C++ compiler): - requirements.windows-test.txt + librephotos/settings/_win_stubs.py + test_windows.py stub the native/ML modules via a meta-path finder. .gitignore: ignore .venv*/ and .testtmp/. CLAUDE.md: document both paths. Co-Authored-By: Claude Opus 4.8 --- apps/backend/.gitignore | 4 + apps/backend/CLAUDE.md | 28 ++++ .../librephotos/settings/_win_stubs.py | 86 +++++++++++++ .../librephotos/settings/test_windows.py | 16 +++ apps/backend/requirements.windows-test.txt | 47 +++++++ apps/backend/requirements.windows.txt | 44 +++++++ apps/backend/scripts/run_tests_windows.ps1 | 72 +++++++++++ apps/backend/scripts/setup_windows.ps1 | 120 ++++++++++++++++++ 8 files changed, 417 insertions(+) create mode 100644 apps/backend/librephotos/settings/_win_stubs.py create mode 100644 apps/backend/librephotos/settings/test_windows.py create mode 100644 apps/backend/requirements.windows-test.txt create mode 100644 apps/backend/requirements.windows.txt create mode 100644 apps/backend/scripts/run_tests_windows.ps1 create mode 100644 apps/backend/scripts/setup_windows.ps1 diff --git a/apps/backend/.gitignore b/apps/backend/.gitignore index dc31ecdb84..a58e455a8f 100644 --- a/apps/backend/.gitignore +++ b/apps/backend/.gitignore @@ -120,12 +120,16 @@ celerybeat.pid # Environments .env .venv +.venv*/ env/ venv/ ENV/ env.bak/ venv.bak/ +# Windows local test runner scratch (see scripts/run_tests_windows.ps1) +.testtmp/ + # Spyder project settings .spyderproject .spyproject diff --git a/apps/backend/CLAUDE.md b/apps/backend/CLAUDE.md index 619c456b24..63d48fb472 100644 --- a/apps/backend/CLAUDE.md +++ b/apps/backend/CLAUDE.md @@ -61,6 +61,34 @@ The `test_sqlite` settings module (`librephotos/settings/test_sqlite.py`) uses an in-memory SQLite database so no PostgreSQL instance is required. +**On Windows (native dev box, no Docker):** the full pinned dependency set installs +natively on Python 3.11 from prebuilt wheels — see `requirements.windows.txt`, which +reuses every pin from `requirements.txt`/`requirements.dev.txt` and only swaps in the +Windows-friendly variants (`pyvips[binary]`, `python-magic-bin`, `psycopg[binary]`, +and the llama-cpp-python CPU wheel index). One-time setup + run: + +```powershell +# requires Python 3.11 (winget install --id Python.Python.3.11) and winget +./scripts/setup_windows.ps1 # venv .venv-win + deps + exiftool/ImageMagick + libmagic shim +./scripts/run_tests_windows.ps1 # whole suite (in-memory SQLite) +./scripts/run_tests_windows.ps1 api.tests.test_photo_metadata +``` + +Notes: `insightface` has no prebuilt wheel and compiles from sdist, so an MSVC v143 +C++ toolchain (Visual Studio 2022 / Build Tools) must be present. `exiftool.exe` is +required (migration 0009 starts it at test-DB setup). `python-magic` needs the bundled +`libmagic.dll` ahead of Git-for-Windows' MSYS one on PATH — `setup_windows.ps1` writes +a venv shim (`_lp_win_magic_shim.pth`) that handles this automatically. `gunicorn` +installs but can't serve on Windows (no `fcntl`); use `manage.py runserver` for a dev +server. + +*Lightweight alternative (no native deps / no C++ compiler):* install +`requirements.windows-test.txt` (pure-Python subset) and run with +`DJANGO_SETTINGS_MODULE=librephotos.settings.test_windows`, which stubs the native/ML +modules (`magic`, `pyvips`, `torch`, `insightface`, ...) via a meta-path finder so the +app imports and the pure-Python logic tests run. Tests that exercise real native +behaviour are expected to fail under the stubs. + ### Debugging - **PDB Breakpoint**: Add `import pdb; pdb.set_trace()` in code - **Attach to Container**: `docker attach $(docker ps --filter name=backend -q)` diff --git a/apps/backend/librephotos/settings/_win_stubs.py b/apps/backend/librephotos/settings/_win_stubs.py new file mode 100644 index 0000000000..e7ec78a894 --- /dev/null +++ b/apps/backend/librephotos/settings/_win_stubs.py @@ -0,0 +1,86 @@ +"""Lightweight import stubs for native / heavy-ML dependencies. + +On a Windows dev box several backend dependencies cannot be pip-installed without +a full native build toolchain or large binary runtimes (libvips, libmagic, +ImageMagick, torch, faiss, hdbscan, insightface, llama-cpp-python, onnxruntime, +the owncloud client, the exiftool binary, ...). + +None of those are needed to exercise the pure-Python API / model / view logic that +the test suite covers. This module installs a meta-path finder that fabricates a +harmless stub module for any import whose top-level package is in ``STUB_ROOTS``. +Each stub returns ``MagicMock`` objects for attribute access, so importing the app +and running tests succeeds; any test that genuinely needs real behaviour from one +of these libraries will fail loudly (a MagicMock where a string/array was expected) +rather than silently — which is the correct signal that the test needs the real +dependency and should run in the Linux container instead. + +Only used by ``librephotos.settings.test_windows``. CI on Linux installs the full +``requirements.txt`` and never imports this. +""" + +import sys +import types +from importlib.abc import Loader, MetaPathFinder +from importlib.machinery import ModuleSpec +from unittest.mock import MagicMock + +# Top-level packages we replace with stubs. Anything under them (e.g. +# ``sklearn.decomposition``, ``torch.nn``) is fabricated on demand by the finder. +STUB_ROOTS = { + "torch", + "transformers", + "sentence_transformers", + "faiss", + "hdbscan", + "sklearn", + "insightface", + "llama_cpp", + "onnxruntime", + "sentencepiece", + "timm", + "magic", + "pyvips", + "owncloud", + "wand", + "cv2", + "exiftool", + "timezonefinder", + "nmslib", + "annoy", +} + + +class _StubModule(types.ModuleType): + def __getattr__(self, name): + # Dunder lookups should behave like a normal module (raise) so the import + # machinery and tools don't get confused by fake __path__/__all__ etc. + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + mock = MagicMock(name=f"{self.__name__}.{name}") + setattr(self, name, mock) + return mock + + +class _StubLoader(Loader): + def create_module(self, spec): + module = _StubModule(spec.name) + module.__path__ = [] # treat as a package so submodule imports resolve + module.__spec__ = spec + return module + + def exec_module(self, module): # noqa: D401 - nothing to execute + pass + + +class _StubFinder(MetaPathFinder): + def find_spec(self, fullname, path, target=None): + root = fullname.split(".")[0] + if root in STUB_ROOTS: + return ModuleSpec(fullname, _StubLoader(), is_package=True) + return None + + +def install(): + """Insert the stub finder once, ahead of the real import machinery.""" + if not any(isinstance(f, _StubFinder) for f in sys.meta_path): + sys.meta_path.insert(0, _StubFinder()) diff --git a/apps/backend/librephotos/settings/test_windows.py b/apps/backend/librephotos/settings/test_windows.py new file mode 100644 index 0000000000..b3a1b2a1f1 --- /dev/null +++ b/apps/backend/librephotos/settings/test_windows.py @@ -0,0 +1,16 @@ +"""Test settings for running the backend suite on a Windows dev box. + +Installs import stubs for native / heavy-ML dependencies that can't be built on +Windows, then reuses the in-memory SQLite test settings. Run with the helper +script ``scripts/run_tests_windows.ps1`` or directly: + + DJANGO_SETTINGS_MODULE=librephotos.settings.test_windows \ + SECRET_KEY=test-secret-key BASE_LOGS=... BASE_DATA=... \ + python manage.py test api.tests +""" + +from ._win_stubs import install as _install_stubs + +_install_stubs() + +from .test_sqlite import * # noqa: E402,F401,F403 diff --git a/apps/backend/requirements.windows-test.txt b/apps/backend/requirements.windows-test.txt new file mode 100644 index 0000000000..832a89c5c3 --- /dev/null +++ b/apps/backend/requirements.windows-test.txt @@ -0,0 +1,47 @@ +# LIGHTWEIGHT / no-compiler Windows test path (the FALLBACK). +# +# Prefer requirements.windows.txt + scripts/setup_windows.ps1, which installs the +# FULL pinned dependency set natively. Use THIS curated subset only when you can't +# or don't want to install the native/ML stack (e.g. no MSVC C++ compiler for +# insightface, or you only need quick pure-Python logic iteration). +# +# Heavy / native-only packages (faiss, hdbscan, insightface, llama-cpp-python, +# pyvips, Wand, python-magic, torch/transformers/sentence_transformers, onnxruntime, +# owncloud, exiftool binary) are NOT installed here. They are replaced at runtime by +# lightweight stub modules (librephotos/settings/_win_stubs.py) so the Django app and +# the test suite can be imported and run against in-memory SQLite. Activate the stubs +# by running with DJANGO_SETTINGS_MODULE=librephotos.settings.test_windows. +# +# Tests that exercise REAL native behaviour will fail under the stubs — that's the +# signal to use the native path instead. CI uses the full requirements.txt in Linux. + +Django==5.2.14 +# binary wheel so django.contrib.postgres can import the driver at app load +# (the test DB itself is in-memory SQLite) +psycopg[binary]==3.3.4 +django-constance==4.3.5 +django-cors-headers==4.9.0 +django-cryptography-5==2.0.3 +django-extensions==4.1 +django-filter==25.2 +django-bulk-update +django-silk==5.5.0 +djangorestframework==3.17.1 +djangorestframework-simplejwt==5.5.1 +drf-spectacular==0.29.0 +Pillow==12.2.0 +ImageHash==4.3.2 +pytz==2026.2 +tzdata==2026.2 +geopy==2.4.1 +nltk==3.9.4 +markupsafe==3.0.3 +tqdm==4.67.3 +diskcache==5.6.3 +py-cpuinfo==9.0.0 +psutil==7.2.2 +concurrent-log-handler==0.9.29 +argon2-cffi==25.1.0 +safetensors==0.7.0 +Faker==40.15.0 +pyfakefs==6.2.0 diff --git a/apps/backend/requirements.windows.txt b/apps/backend/requirements.windows.txt new file mode 100644 index 0000000000..fa6e9e4834 --- /dev/null +++ b/apps/backend/requirements.windows.txt @@ -0,0 +1,44 @@ +# Windows-native install of the FULL backend dependency set (Python 3.11+). +# +# This reuses every pin from requirements.txt and only adds the Windows-specific +# adjustments needed to install the "hostile" native/ML packages from prebuilt +# wheels instead of building them from source. Verified on Windows 11 / +# CPython 3.11.9 (win_amd64) — every package installs and imports. +# +# py -3.11 -m venv apps/backend/.venv-win +# apps/backend/.venv-win/Scripts/python -m pip install -U pip wheel +# apps/backend/.venv-win/Scripts/python -m pip install -r apps/backend/requirements.windows.txt +# +# Two dependencies also need an external (non-pip) binary; install them with +# scripts/setup_windows.ps1 (winget), which also wires up the libmagic DLL shim: +# - Wand -> ImageMagick (winget id ImageMagick.ImageMagick — the inno DLL build) +# - PyExifTool -> exiftool.exe (winget id OliverBetz.ExifTool, runtime only) +# +# llama-cpp-python ships only an sdist on PyPI (CMake/C++ source build); the +# --extra-index-url below pulls the maintainer's prebuilt CPU wheel instead. +--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu + +# All canonical pins (Django, DRF, torch stack, faiss, onnxruntime, hdbscan, +# insightface, gevent, pyocclient, py-cpuinfo, ...) come straight from here so +# this file never drifts out of sync with the source of truth. +-r requirements.txt + +# Dev/test tooling (Faker, pyfakefs, coverage, pytest, ruff, ...) — needed to run +# the test suite. All pure-Python, installs cleanly on Windows. +-r requirements.dev.txt + +# --- Windows-specific additions / overrides ------------------------------------ +# psycopg: requirements.txt pins the pure `psycopg` (it relies on the system +# libpq present in the Linux container). Windows has no system libpq, so pull the +# self-contained binary build. (The test suite uses SQLite, but django.contrib. +# postgres still imports the driver at app load.) +psycopg[binary]==3.3.4 + +# pyvips: pull the bundled-libvips wheel (pyvips-binary) so no system libvips is +# needed. Unions with the pyvips==3.1.1 pin from requirements.txt. +pyvips[binary]==3.1.1 + +# python-magic: ship the native 64-bit libmagic.dll + magic.mgc. NOTE: import +# still needs the bundled DLL dir ahead of Git-for-Windows' MSYS libmagic on +# PATH — setup_windows.ps1 installs a sitecustomize shim that handles this. +python-magic-bin==0.4.14 diff --git a/apps/backend/scripts/run_tests_windows.ps1 b/apps/backend/scripts/run_tests_windows.ps1 new file mode 100644 index 0000000000..e43b4d5f4e --- /dev/null +++ b/apps/backend/scripts/run_tests_windows.ps1 @@ -0,0 +1,72 @@ +<# +.SYNOPSIS + Run the LibrePhotos backend test suite on a Windows dev box (in-memory SQLite). + +.DESCRIPTION + Uses the NATIVE Windows dev venv at apps/backend/.venv-win -- the full pinned + dependency set installed from requirements.windows.txt (real pyvips, libmagic, + torch, insightface, exiftool, ...). Run scripts/setup_windows.ps1 once first. + + The test DB is in-memory SQLite (librephotos.settings.test_sqlite); no Postgres + needed. The libmagic loader shim is applied automatically by the venv (see + setup_windows.ps1); this script additionally puts exiftool.exe on PATH, which + migration 0009 needs at test-DB setup. + + Lightweight alternative (no native deps / no C++ compiler): install + requirements.windows-test.txt and set DJANGO_SETTINGS_MODULE to + librephotos.settings.test_windows (stubs the native modules). See CLAUDE.md. + +.PARAMETER TestArgs + Arguments passed straight to `manage.py test`, e.g. a dotted test path like + api.tests.test_photo_lifecycle, optionally with flags such as `-v 2`. + Defaults to the whole suite (api.tests). + +.EXAMPLE + ./scripts/run_tests_windows.ps1 + ./scripts/run_tests_windows.ps1 api.tests.test_multi_user_isolation + ./scripts/run_tests_windows.ps1 api.tests.test_stats_accuracy -v 2 +#> +param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$TestArgs = @("api.tests") +) + +if (-not $TestArgs -or $TestArgs.Count -eq 0) { $TestArgs = @("api.tests") } + +$ErrorActionPreference = "Stop" +$BackendDir = Split-Path -Parent $PSScriptRoot +$Python = Join-Path $BackendDir ".venv-win\Scripts\python.exe" + +if (-not (Test-Path $Python)) { + Write-Error "venv not found at $Python. Run ./scripts/setup_windows.ps1 first." + exit 1 +} + +$TmpRoot = Join-Path $BackendDir ".testtmp" +foreach ($sub in @("logs", "data", "protected_media")) { + New-Item -ItemType Directory -Force -Path (Join-Path $TmpRoot $sub) | Out-Null +} + +# Ensure exiftool.exe is reachable (winget installs it here for the current user). +$ExifDir = Join-Path $env:LOCALAPPDATA "Programs\ExifTool" +if ((Test-Path $ExifDir) -and ($env:PATH -notlike "*$ExifDir*")) { + $env:PATH = "$ExifDir;$env:PATH" +} +if (-not (Get-Command exiftool -ErrorAction SilentlyContinue)) { + Write-Warning "exiftool.exe not found on PATH -- migration 0009 will fail. Install: winget install --id OliverBetz.ExifTool" +} + +$env:DJANGO_SETTINGS_MODULE = "librephotos.settings.test_sqlite" +$env:SECRET_KEY = "test-secret-key" +$env:BASE_DATA = $TmpRoot +$env:BASE_LOGS = Join-Path $TmpRoot "logs" +$env:NO_COVERAGE = "1" + +Push-Location $BackendDir +try { + & $Python manage.py test @TestArgs + exit $LASTEXITCODE +} +finally { + Pop-Location +} diff --git a/apps/backend/scripts/setup_windows.ps1 b/apps/backend/scripts/setup_windows.ps1 new file mode 100644 index 0000000000..49874bcba9 --- /dev/null +++ b/apps/backend/scripts/setup_windows.ps1 @@ -0,0 +1,120 @@ +<# +.SYNOPSIS + One-time setup of a NATIVE LibrePhotos backend dev environment on Windows. + +.DESCRIPTION + Installs the full pinned dependency set from requirements.windows.txt into a + Python 3.11 venv at apps/backend/.venv-win, installs the two external binaries + that two packages need (exiftool for PyExifTool, ImageMagick for Wand), and + writes a small libmagic loader shim into the venv so python-magic imports the + correct DLL. + + Everything was verified on Windows 11 / CPython 3.11.9 (win_amd64). After this + finishes, run the suite with: ./scripts/run_tests_windows.ps1 + +.NOTES + - Requires Python 3.11 (winget install Python.Python.3.11) and winget. + - insightface has no prebuilt wheel and is compiled from sdist; that needs the + MSVC v143 C++ toolchain. If Visual Studio / Build Tools is absent the script + warns and tells you how to install it. (If you can't install a compiler, use + the lightweight stub path instead -- see apps/backend/CLAUDE.md.) +#> +[CmdletBinding()] +param( + [switch]$SkipExternalBinaries +) + +$ErrorActionPreference = "Stop" +$BackendDir = Split-Path -Parent $PSScriptRoot +$VenvDir = Join-Path $BackendDir ".venv-win" +$Reqs = Join-Path $BackendDir "requirements.windows.txt" + +function Write-Step($m) { Write-Host "`n=== $m ===" -ForegroundColor Cyan } + +# --- 1. Locate Python 3.11 ----------------------------------------------------- +Write-Step "Locating Python 3.11" +$py = $null +try { & py -3.11 --version *> $null; if ($LASTEXITCODE -eq 0) { $py = @("py", "-3.11") } } catch {} +if (-not $py) { + Write-Error "Python 3.11 not found. Install it with: winget install --id Python.Python.3.11 --scope user" + exit 1 +} +Write-Host ("Using: " + (& $py[0] $py[1] -c "import sys; print(sys.executable)")) + +# --- 2. Create venv ------------------------------------------------------------ +Write-Step "Creating venv at $VenvDir" +if (-not (Test-Path (Join-Path $VenvDir "Scripts\python.exe"))) { + & $py[0] $py[1] -m venv $VenvDir +} +$VenvPy = Join-Path $VenvDir "Scripts\python.exe" +& $VenvPy -m pip install -U pip wheel | Out-Null + +# --- 3. Install the full pinned dependency set --------------------------------- +Write-Step "Installing requirements.windows.txt (this pulls torch, faiss, insightface, ... -- several minutes)" +& $VenvPy -m pip install -r $Reqs +if ($LASTEXITCODE -ne 0) { + Write-Warning "pip install failed. If the failure was building 'insightface', you are missing the MSVC C++ compiler." + Write-Warning "Install it with: winget install --id Microsoft.VisualStudio.2022.BuildTools --override `"--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended --quiet`"" + exit 1 +} + +# --- 4. libmagic loader shim --------------------------------------------------- +# python-magic on Windows can load Git-for-Windows' MSYS libmagic (msys-magic-1.dll) +# which crashes CPython. Prepend python-magic-bin's bundled DLL dir at startup. +Write-Step "Installing libmagic loader shim into the venv" +$SitePackages = & $VenvPy -c "import sysconfig; print(sysconfig.get_paths()['purelib'])" +$ShimPy = Join-Path $SitePackages "_lp_win_magic_shim.py" +$ShimPth = Join-Path $SitePackages "_lp_win_magic_shim.pth" +@' +"""LibrePhotos Windows dev shim: make python-magic load the RIGHT libmagic. + +Git-for-Windows' usr\bin on PATH provides an MSYS/cygwin libmagic that crashes +native CPython when python-magic loads it. Prepend python-magic-bin's bundled +native libmagic.dll directory at interpreter startup so it wins. No-op elsewhere. +Loaded automatically via the companion _lp_win_magic_shim.pth file. +""" +import os +import sys + + +def _install(): + if sys.platform != "win32": + return + try: + import importlib.util + + spec = importlib.util.find_spec("magic") + if not spec or not spec.origin: + return + libmagic_dir = os.path.join(os.path.dirname(spec.origin), "libmagic") + if os.path.isdir(libmagic_dir): + os.environ["PATH"] = libmagic_dir + os.pathsep + os.environ.get("PATH", "") + except Exception: + pass + + +_install() +'@ | Set-Content -Encoding UTF8 $ShimPy +"import _lp_win_magic_shim`n" | Set-Content -Encoding UTF8 $ShimPth +Write-Host "Wrote $ShimPy" + +# --- 5. External binaries ------------------------------------------------------ +if (-not $SkipExternalBinaries) { + Write-Step "Installing external binaries via winget (exiftool, ImageMagick)" + # exiftool.exe: REQUIRED -- migration 0009 starts an ExifTool process at test-DB setup. + winget install --id OliverBetz.ExifTool --silent --accept-package-agreements --accept-source-agreements + # ImageMagick: needed only by Wand (the inno DLL build -- NOT the msix single-exe). + winget install --id ImageMagick.ImageMagick --silent --accept-package-agreements --accept-source-agreements + Write-Host "NOTE: open a fresh shell so exiftool.exe / ImageMagick land on PATH." +} else { + Write-Host "`nSkipped external binaries. exiftool.exe is REQUIRED for the test suite (winget install --id OliverBetz.ExifTool)." +} + +# --- 6. MSVC sanity note ------------------------------------------------------- +$vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" +if (-not (Test-Path $vswhere)) { + Write-Warning "No Visual Studio detected. insightface compiles from source and needs the MSVC v143 C++ toolchain if it wasn't already installed above." +} + +Write-Step "Done" +Write-Host "Run the test suite with: ./scripts/run_tests_windows.ps1" -ForegroundColor Green