diff --git a/README.md b/README.md index 10ac74b..94d8ff5 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,34 @@ journalctl -u scanfiler.service -f macOS/Windows: use `scanfiler loop` under launchd / Task Scheduler. +## Self-update + +With `auto_update.enabled: true`, each run first checks the git remote and, if a newer +version exists, updates and re-runs on it before doing any work: + +``` +git fetch → pick target (latest release tag by default, or a branch) + → if newer & working tree clean: checkout → pip install (if pyproject.toml changed) + → re-exec into the new version (so this run uses it) +``` + +- **`ref`** — `latest-release` (newest `vX.Y.Z` tag; stable, recommended) or a branch + name like `main` (bleeding edge). +- **Signature verification (on by default).** Before applying, the target commit's + signature is checked with `git verify-commit`. If it isn't validly signed by a trusted + key, the update is **refused** (fail-closed) and the run continues on the current + version. Disable with `verify_signature: false` only if you understand the risk. For + SSH-signed commits, point `allowed_signers_file` at an allowed-signers file; for GPG, + import your public key into the deploy user's keyring and leave it unset. +- **Fail-safe.** Any problem (offline, dirty tree, install error) is logged and the run + continues on the current version; a failed install rolls the checkout back. +- **Skipped** when not run from a git clone, when the working tree is dirty, and on + `--dry-run`. In `loop` mode the check runs each cycle, so a long-running daemon picks + up releases without a manual restart (it tracks tags via a detached HEAD). +- **Requires the deploy to be a git working tree** with network access to the remote — + i.e. `git clone` the repo and `pip install -e .` rather than installing a wheel. The + `repo_dir` defaults to the repo root above the package; override it if needed. + ## Testing ```bash diff --git a/config.example.yaml b/config.example.yaml index 000c4af..5da89e2 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -53,6 +53,15 @@ logging: audit_file: ./logs/audit.jsonl # JSONL record of every move (reversible via undo) ledger_db: ./state/ledger.sqlite # content-hash processed-file ledger +auto_update: + enabled: false # check for + apply a newer version before each run + ref: latest-release # newest vX.Y.Z tag, or a branch name (e.g. main) + install_deps: true # reinstall (pip install -e .) when pyproject.toml changed + restart: true # re-exec into the new version so this run uses it + verify_signature: true # require a trusted commit signature (fail-closed) + # allowed_signers_file: /etc/scanfiler/allowed_signers # SSH-signed commits; omit for gpg keyring + # repo_dir: /opt/scanfiler # defaults to the repo root above the package + prompt: context: > These are personal scanned documents: receipts, invoices, medical records, diff --git a/scanfiler/cli.py b/scanfiler/cli.py index aa3dffc..b7f323c 100644 --- a/scanfiler/cli.py +++ b/scanfiler/cli.py @@ -78,6 +78,14 @@ def _run_once(cfg: Config, args) -> int: from .pipeline import plan from .proposals import write_proposals + # Check for + apply a newer release before doing any work. Done before the lock so a + # re-exec into the updated version doesn't deadlock against our own lockfile. On a + # dry run we never mutate the install. + if cfg.auto_update.enabled and not args.dry_run: + from .self_update import default_deps, perform_self_update + + perform_self_update(cfg.auto_update, default_deps(cfg.auto_update)) + client = _make_client(cfg) lock_path = Path(cfg.logging.ledger_db).with_suffix(".lock") try: diff --git a/scanfiler/config.py b/scanfiler/config.py index 82c50f6..c94b144 100644 --- a/scanfiler/config.py +++ b/scanfiler/config.py @@ -101,6 +101,17 @@ class PromptConfig(BaseModel): ) +class AutoUpdateConfig(BaseModel): + enabled: bool = False + # 'latest-release' (newest vX.Y.Z tag) or a branch name (e.g. main) to track. + ref: str = "latest-release" + install_deps: bool = True # reinstall when pyproject.toml changed + restart: bool = True # re-exec into the new version this run + verify_signature: bool = True # require a trusted commit signature (fail-closed) + allowed_signers_file: str | None = None # SSH allowed-signers file; else system gpg trust + repo_dir: str | None = None # defaults to the repo root above the package + + class Config(BaseModel): paths: PathsConfig ai: AIConfig = Field(default_factory=AIConfig) @@ -113,6 +124,7 @@ class Config(BaseModel): apply: ApplyConfig = Field(default_factory=ApplyConfig) logging: LoggingConfig = Field(default_factory=LoggingConfig) prompt: PromptConfig = Field(default_factory=PromptConfig) + auto_update: AutoUpdateConfig = Field(default_factory=AutoUpdateConfig) @field_validator("paths") @classmethod diff --git a/scanfiler/self_update.py b/scanfiler/self_update.py new file mode 100644 index 0000000..9274db3 --- /dev/null +++ b/scanfiler/self_update.py @@ -0,0 +1,222 @@ +"""Self-update: check for a newer release and update before a run. + +Mirrors the actual-ai-categorizer model. Self-contained and fail-safe: any failure is +logged and the run continues on the current version. If an update is applied and +`restart` is set, the process re-execs into the new code so this very run uses it. + +Only works when running from a git working tree (the recommended deploy is a git +clone); a wheel install in site-packages is skipped with a warning. Dependencies are +injected (run/reexec/env/log) so the logic is unit-testable without real git. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import NoReturn + +from .config import AutoUpdateConfig + +# Set on the re-exec'd child so it doesn't immediately try to update again (which would +# loop). Cleared after the first cycle so later loop iterations still check for updates. +SENTINEL = "SCANFILER_SELF_UPDATED" + + +@dataclass +class SelfUpdateDeps: + repo_dir: str + run: Callable[[str, list[str]], str] # run in repo dir, return stdout, raise on error + reexec: Callable[[], NoReturn] # re-exec current process; never returns + env: dict + log: Callable[[str, str], None] # (level, message) + + +@dataclass +class _Target: + label: str + commit: str + checkout: Callable[[], None] + + +def perform_self_update(cfg: AutoUpdateConfig, deps: SelfUpdateDeps) -> None: + if not cfg.enabled: + return + + if deps.env.get(SENTINEL) == "1": + deps.log("debug", "self-update: running freshly updated version; skipping check") + deps.env.pop(SENTINEL, None) + return + + updated = False + try: + if not _is_git_repo(deps): + deps.log("warn", f"self-update: {deps.repo_dir} is not a git repository; skipping") + return + + deps.run("git", ["fetch", "--tags", "--prune", "--quiet", "origin"]) + old_head = _git(deps, ["rev-parse", "HEAD"]) + target = _resolve_target(cfg, deps) + if target is None: + return # _resolve_target already logged why + + if target.commit == old_head: + deps.log("info", f"self-update: already up to date ({target.label})") + return + if not _is_clean_tree(deps): + deps.log("warn", "self-update: working tree has uncommitted changes; skipping") + return + if cfg.verify_signature and not _verify_commit(deps, cfg, target.commit): + deps.log( + "error", + f"self-update: signature verification FAILED for {target.label} " + f"({target.commit[:8]}); refusing to update", + ) + return + + deps.log("info", f"self-update: updating to {target.label}") + target.checkout() + updated = True + + try: + if cfg.install_deps and _pyproject_changed(deps, old_head, target.commit): + deps.log("info", "self-update: dependencies changed; reinstalling") + deps.run(sys.executable, ["-m", "pip", "install", "-e", ".", "--quiet"]) + except Exception as post_err: # noqa: BLE001 + deps.log( + "error", + f"self-update: post-update step failed ({post_err}); " + f"rolling back to {old_head[:8]}", + ) + _rollback(deps, old_head) + return + except Exception as err: # noqa: BLE001 + deps.log("warn", f"self-update: check failed ({err}); continuing with current version") + return + + if updated and cfg.restart: + deps.log("info", "self-update: restarting into the updated version") + deps.reexec() # never returns + + +def _git(deps: SelfUpdateDeps, args: list[str]) -> str: + return deps.run("git", args).strip() + + +def _is_git_repo(deps: SelfUpdateDeps) -> bool: + try: + return _git(deps, ["rev-parse", "--is-inside-work-tree"]) == "true" + except Exception: # noqa: BLE001 + return False + + +def _is_clean_tree(deps: SelfUpdateDeps) -> bool: + return _git(deps, ["status", "--porcelain"]) == "" + + +def _verify_commit(deps: SelfUpdateDeps, cfg: AutoUpdateConfig, commit: str) -> bool: + args: list[str] = [] + if cfg.allowed_signers_file: + args += ["-c", "gpg.format=ssh", + "-c", f"gpg.ssh.allowedSignersFile={cfg.allowed_signers_file}"] + args += ["verify-commit", "--raw", commit] + try: + deps.run("git", args) + deps.log("info", f"self-update: signature verified for {commit[:8]}") + return True + except Exception: # noqa: BLE001 + return False + + +def _pyproject_changed(deps: SelfUpdateDeps, frm: str, to: str) -> bool: + try: + return _git(deps, ["diff", "--name-only", frm, to, "--", "pyproject.toml"]) != "" + except Exception: # noqa: BLE001 + return True # can't tell -> reinstall to be safe + + +def _resolve_target(cfg: AutoUpdateConfig, deps: SelfUpdateDeps) -> _Target | None: + if cfg.ref == "latest-release": + tags = [ + t.strip() + for t in _git(deps, ["tag", "-l", "v*.*.*", "--sort=-v:refname"]).splitlines() + if t.strip() + ] + if not tags: + deps.log("info", "self-update: no release tags found; skipping") + return None + tag = tags[0] + return _Target( + label=tag, + commit=_git(deps, ["rev-parse", f"{tag}^{{commit}}"]), + checkout=lambda: deps.run( + "git", ["-c", "advice.detachedHead=false", "checkout", "--quiet", tag] + ), + ) + + branch = cfg.ref + + def _checkout_branch() -> None: + deps.run("git", ["checkout", "--quiet", branch]) + deps.run("git", ["merge", "--ff-only", "--quiet", f"origin/{branch}"]) + + return _Target( + label=branch, + commit=_git(deps, ["rev-parse", f"origin/{branch}"]), + checkout=_checkout_branch, + ) + + +def _rollback(deps: SelfUpdateDeps, old_head: str) -> None: + try: + deps.run("git", ["-c", "advice.detachedHead=false", "checkout", "--quiet", old_head]) + except Exception: # noqa: BLE001 + pass # nothing more we can do; current in-memory code still runs this cycle + + +# ---- default (production) dependency implementations ---- + +def resolve_repo_dir(override: str | None = None) -> str: + """Repo root: explicit override, else the directory above this package.""" + if override: + return override + return str(Path(__file__).resolve().parent.parent) + + +def default_run(repo_dir: str) -> Callable[[str, list[str]], str]: + def _run(cmd: str, args: list[str]) -> str: + return subprocess.run( + [cmd, *args], + cwd=repo_dir, + check=True, + capture_output=True, + text=True, + ).stdout + return _run + + +def default_reexec(env: dict) -> NoReturn: + """Re-run `python -m scanfiler ` with the sentinel set, then exit.""" + child_env = {**env, SENTINEL: "1"} + argv = [sys.executable, "-m", "scanfiler", *sys.argv[1:]] + result = subprocess.run(argv, env=child_env) + sys.exit(result.returncode) + + +def _default_log(level: str, message: str) -> None: + stream = sys.stderr if level in ("warn", "error") else sys.stdout + print(message, file=stream) + + +def default_deps(cfg: AutoUpdateConfig) -> SelfUpdateDeps: + repo_dir = resolve_repo_dir(cfg.repo_dir) + return SelfUpdateDeps( + repo_dir=repo_dir, + run=default_run(repo_dir), + reexec=lambda: default_reexec(dict(os.environ)), + env=os.environ, + log=_default_log, + ) diff --git a/scanfiler/templates.py b/scanfiler/templates.py index d1d8a6d..46592c4 100644 --- a/scanfiler/templates.py +++ b/scanfiler/templates.py @@ -56,6 +56,15 @@ audit_file: ./logs/audit.jsonl # JSONL record of every move (reversible via undo) ledger_db: ./state/ledger.sqlite # content-hash processed-file ledger +auto_update: + enabled: false # check for + apply a newer version before each run + ref: latest-release # newest vX.Y.Z tag, or a branch name (e.g. main) + install_deps: true # reinstall (pip install -e .) when pyproject.toml changed + restart: true # re-exec into the new version so this run uses it + verify_signature: true # require a trusted commit signature (fail-closed) + # allowed_signers_file: /etc/scanfiler/allowed_signers # SSH-signed commits; omit for gpg keyring + # repo_dir: /opt/scanfiler # defaults to the repo root above the package + prompt: context: > These are personal scanned documents: receipts, invoices, medical records, diff --git a/tests/test_cli.py b/tests/test_cli.py index 7bd3f41..a3b6f8f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -87,6 +87,34 @@ def test_undo_nonexistent_run(config_file, capsys): assert "restored 0" in capsys.readouterr().out +def test_run_triggers_self_update_when_enabled(config_file, monkeypatch): + import scanfiler.self_update as su + + called = {"n": 0} + monkeypatch.setattr(su, "perform_self_update", lambda cfg, deps: called.__setitem__("n", 1)) + + data = __import__("yaml").safe_load(config_file.read_text(encoding="utf-8")) + data["auto_update"] = {"enabled": True, "verify_signature": False} + config_file.write_text(__import__("yaml").safe_dump(data), encoding="utf-8") + + cli.main(["-c", str(config_file), "run"]) + assert called["n"] == 1 + + +def test_dry_run_skips_self_update(config_file, monkeypatch): + import scanfiler.self_update as su + + called = {"n": 0} + monkeypatch.setattr(su, "perform_self_update", lambda cfg, deps: called.__setitem__("n", 1)) + + data = __import__("yaml").safe_load(config_file.read_text(encoding="utf-8")) + data["auto_update"] = {"enabled": True} + config_file.write_text(__import__("yaml").safe_dump(data), encoding="utf-8") + + cli.main(["-c", str(config_file), "run", "--dry-run"]) + assert called["n"] == 0 + + def test_loop_stops_on_keyboard_interrupt(config_file, monkeypatch, capsys): def stop(*a, **k): raise KeyboardInterrupt diff --git a/tests/test_self_update.py b/tests/test_self_update.py new file mode 100644 index 0000000..833032d --- /dev/null +++ b/tests/test_self_update.py @@ -0,0 +1,278 @@ +"""Self-update logic tests with a fake git/command runner (no real git or network).""" + +from __future__ import annotations + +import subprocess +import sys + +import pytest + +from scanfiler.config import AutoUpdateConfig +from scanfiler.self_update import ( + SENTINEL, + SelfUpdateDeps, + default_deps, + default_reexec, + default_run, + perform_self_update, + resolve_repo_dir, +) + + +class FakeGit: + """Scriptable stand-in for the injected `run`; records every invocation.""" + + def __init__( + self, + *, + is_repo=True, + head="oldsha", + tags=("v0.2.0",), + tag_commit="newsha", + origin_commit="newsha", + clean=True, + verify_ok=True, + pyproject_changed=True, + fetch_raises=False, + fail=(), # substrings; any command whose args contain one raises + ): + self.is_repo = is_repo + self.head = head + self.tags = list(tags) + self.tag_commit = tag_commit + self.origin_commit = origin_commit + self.clean = clean + self.verify_ok = verify_ok + self.pyproject_changed = pyproject_changed + self.fetch_raises = fetch_raises + self.fail = fail + self.calls: list[tuple[str, list[str]]] = [] + + def run(self, cmd: str, args: list[str]) -> str: + self.calls.append((cmd, args)) + if any(f in args for f in self.fail): + raise RuntimeError(f"command failed: {args}") + + if cmd != "git": # e.g. pip install + return "" + if "--is-inside-work-tree" in args: + if not self.is_repo: + raise RuntimeError("not a repo") + return "true\n" + if "fetch" in args: + if self.fetch_raises: + raise RuntimeError("network down") + return "" + if args[:2] == ["rev-parse", "HEAD"]: + return self.head + "\n" + if "rev-parse" in args and any(a.endswith("^{commit}") for a in args): + return self.tag_commit + "\n" + if "rev-parse" in args and any(a.startswith("origin/") for a in args): + return self.origin_commit + "\n" + if "tag" in args: + return "\n".join(self.tags) + "\n" + if "status" in args: + return "" if self.clean else " M file\n" + if "verify-commit" in args: + if not self.verify_ok: + raise RuntimeError("bad signature") + return "" + if "diff" in args: + return "pyproject.toml\n" if self.pyproject_changed else "" + if "checkout" in args or "merge" in args: + return "" + return "" + + +def _deps(fake: FakeGit, env=None): + state = {"reexec": False} + + def reexec(): + state["reexec"] = True + + logs: list[tuple[str, str]] = [] + deps = SelfUpdateDeps( + repo_dir="/repo", + run=fake.run, + reexec=reexec, + env=env if env is not None else {}, + log=lambda level, msg: logs.append((level, msg)), + ) + return deps, state, logs + + +def _did(fake, verb): + return any(verb in args for _, args in fake.calls) + + +def test_disabled_does_nothing(): + fake = FakeGit() + deps, state, _ = _deps(fake) + perform_self_update(AutoUpdateConfig(enabled=False), deps) + assert fake.calls == [] + assert state["reexec"] is False + + +def test_sentinel_skips_and_clears(): + fake = FakeGit() + env = {SENTINEL: "1"} + deps, state, _ = _deps(fake, env=env) + perform_self_update(AutoUpdateConfig(enabled=True), deps) + assert fake.calls == [] # no git work + assert SENTINEL not in env # cleared for next cycle + + +def test_not_a_repo_skips(): + fake = FakeGit(is_repo=False) + deps, state, logs = _deps(fake) + perform_self_update(AutoUpdateConfig(enabled=True), deps) + assert not _did(fake, "fetch") + assert any(lvl == "warn" for lvl, _ in logs) + + +def test_up_to_date_no_checkout(): + fake = FakeGit(head="samesha", tag_commit="samesha") + deps, state, logs = _deps(fake) + perform_self_update(AutoUpdateConfig(enabled=True, verify_signature=False), deps) + assert not _did(fake, "checkout") + assert state["reexec"] is False + + +def test_dirty_tree_skips(): + fake = FakeGit(clean=False) + deps, state, logs = _deps(fake) + perform_self_update(AutoUpdateConfig(enabled=True, verify_signature=False), deps) + assert not _did(fake, "checkout") + + +def test_signature_failure_refuses(): + fake = FakeGit(verify_ok=False) + deps, state, logs = _deps(fake) + perform_self_update(AutoUpdateConfig(enabled=True, verify_signature=True), deps) + assert not _did(fake, "checkout") + assert any(lvl == "error" for lvl, _ in logs) + + +def test_successful_update_reexecs_and_reinstalls(): + fake = FakeGit(pyproject_changed=True) + deps, state, logs = _deps(fake) + perform_self_update(AutoUpdateConfig(enabled=True, verify_signature=False), deps) + assert _did(fake, "checkout") + assert any(cmd == sys.executable for cmd, _ in fake.calls) # pip install ran + assert state["reexec"] is True + + +def test_update_skips_reinstall_when_pyproject_unchanged(): + fake = FakeGit(pyproject_changed=False) + deps, state, _ = _deps(fake) + perform_self_update(AutoUpdateConfig(enabled=True, verify_signature=False), deps) + assert not any(cmd == sys.executable for cmd, _ in fake.calls) + assert state["reexec"] is True + + +def test_restart_false_no_reexec(): + fake = FakeGit() + deps, state, _ = _deps(fake) + perform_self_update( + AutoUpdateConfig(enabled=True, verify_signature=False, restart=False), deps + ) + assert _did(fake, "checkout") + assert state["reexec"] is False + + +def test_post_step_failure_rolls_back(): + # pip install fails -> rollback checks out the old HEAD, no reexec. + fake = FakeGit(pyproject_changed=True, fail=("install",)) + deps, state, logs = _deps(fake) + perform_self_update(AutoUpdateConfig(enabled=True, verify_signature=False), deps) + assert state["reexec"] is False + assert any("oldsha" in args for _, args in fake.calls) # rollback checkout + assert any(lvl == "error" for lvl, _ in logs) + + +def test_branch_ref_uses_origin_and_merges(): + fake = FakeGit(origin_commit="branchsha") + deps, state, _ = _deps(fake) + perform_self_update( + AutoUpdateConfig(enabled=True, verify_signature=False, ref="main"), deps + ) + assert _did(fake, "merge") + assert state["reexec"] is True + + +def test_no_tags_skips(): + fake = FakeGit(tags=()) + deps, state, logs = _deps(fake) + perform_self_update(AutoUpdateConfig(enabled=True, verify_signature=False), deps) + assert not _did(fake, "checkout") + assert any("no release tags" in msg for _, msg in logs) + + +def test_fetch_error_continues(): + fake = FakeGit(fetch_raises=True) + deps, state, logs = _deps(fake) + perform_self_update(AutoUpdateConfig(enabled=True), deps) + assert state["reexec"] is False + assert any(lvl == "warn" for lvl, _ in logs) + + +def test_allowed_signers_file_passed_to_verify(): + fake = FakeGit() + deps, state, _ = _deps(fake) + perform_self_update( + AutoUpdateConfig(enabled=True, verify_signature=True, + allowed_signers_file="/etc/scanfiler/allowed_signers"), + deps, + ) + verify_call = next(args for cmd, args in fake.calls if "verify-commit" in args) + assert "gpg.format=ssh" in verify_call + assert any("allowedSignersFile=/etc/scanfiler/allowed_signers" in a for a in verify_call) + + +# ---- default helpers ---- + +def test_resolve_repo_dir_override_and_default(): + from pathlib import Path + + assert resolve_repo_dir("/custom") == "/custom" + # Default is the repo root above the package, so it contains the scanfiler/ package. + default = resolve_repo_dir() + assert (Path(default) / "scanfiler" / "self_update.py").is_file() + + +def test_default_run_executes(tmp_path): + run = default_run(str(tmp_path)) + out = run(sys.executable, ["-c", "print('hello-run')"]) + assert "hello-run" in out + + +def test_default_run_raises_on_failure(tmp_path): + run = default_run(str(tmp_path)) + with pytest.raises(subprocess.CalledProcessError): + run(sys.executable, ["-c", "import sys; sys.exit(3)"]) + + +def test_default_reexec_sets_sentinel_and_exits(monkeypatch): + captured = {} + + class _Res: + returncode = 0 + + def fake_run(argv, env=None): + captured["argv"] = argv + captured["env"] = env + return _Res() + + monkeypatch.setattr("scanfiler.self_update.subprocess.run", fake_run) + monkeypatch.setattr(sys, "argv", ["scanfiler", "run", "-c", "config.yaml"]) + with pytest.raises(SystemExit): + default_reexec({"PATH": "x"}) + assert captured["argv"][:3] == [sys.executable, "-m", "scanfiler"] + assert captured["argv"][3:] == ["run", "-c", "config.yaml"] + assert captured["env"][SENTINEL] == "1" + + +def test_default_deps_wired(): + deps = default_deps(AutoUpdateConfig(repo_dir="/repo")) + assert deps.repo_dir == "/repo" + assert callable(deps.run) and callable(deps.reexec)