diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b1cb4f9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,87 @@ +name: CI + +# Gates run on every PR and every push to main. A release is cut only when a push +# to main passes all gates. +on: + push: + branches: [main] + pull_request: + branches: [main] + +# Don't let overlapping runs race on the release/tag step. PR runs supersede each +# other; runs on main are never cancelled mid-release. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + gate: + name: Gate (Python ${{ matrix.python }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ['3.11', '3.12', '3.13'] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + cache: pip + + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Lint (ruff) + run: ruff check . + + - name: Test (pytest + coverage gate) + run: pytest + + - name: Build (sdist + wheel) + run: python -m build + + release: + name: Release + needs: [gate] + # Only on direct pushes to main (not PRs, not forks). + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: write # create tags and releases + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # need full tag history to compute the next version + + - name: Determine next version + id: ver + run: | + latest=$(git tag -l 'v*.*.*' --sort=-v:refname | head -n1) + if [ -z "$latest" ]; then + next="v0.1.0" + else + ver=${latest#v} + IFS='.' read -r major minor patch <<< "$ver" + next="v${major}.${minor}.$((patch + 1))" + fi + echo "Next release: $next (previous: ${latest:-none})" + echo "next=$next" >> "$GITHUB_OUTPUT" + + - name: Create tag and GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + next="${{ steps.ver.outputs.next }}" + git tag "$next" + git push origin "$next" + gh release create "$next" \ + --title "$next" \ + --target "${{ github.sha }}" \ + --generate-notes diff --git a/.gitignore b/.gitignore index a706a05..cc67c65 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,13 @@ __pycache__/ build/ dist/ +# Test / coverage artifacts +.coverage +.coverage.* +htmlcov/ +.pytest_cache/ +.ruff_cache/ + # Runtime state / secrets / user config config.yaml secrets.env @@ -14,6 +21,13 @@ proposals.jsonl /state/ *.lock +# Sample demo: keep the input documents + demo config, ignore generated outputs. +!samples/config.yaml +/samples/library/ +/samples/state/ +/samples/logs/ +/samples/proposals.jsonl + # OS .DS_Store Thumbs.db diff --git a/README.md b/README.md index ee6b0d3..10ac74b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # scanfiler +[![CI](https://github.com/ridaken/Scanfiler/actions/workflows/ci.yml/badge.svg)](https://github.com/ridaken/Scanfiler/actions/workflows/ci.yml) + AI-powered renamer / reorganizer for a dump of scanned documents. It walks an inbox **one file at a time**, sends the first page(s) to an OpenAI-compatible vision-language model (built for [llama.cpp / llama-server](https://github.com/ggml-org/llama.cpp) and @@ -53,6 +55,23 @@ scanfiler undo --last # reverse the most recent apply run scanfiler --dry-run # decide + log, never touch disk ``` +## Try it on the bundled samples + +The repo ships three sample inputs in `samples/inbox/` (an auto-service receipt PDF, +an electrician's invoice docx, and a child's crayon drawing PNG) and a ready-to-run +`samples/config.yaml`. Point `ai.base_url` at a vision model, then: + +```bash +scanfiler -c samples/config.yaml plan --proposals samples/proposals.jsonl +# review samples/proposals.jsonl, then: +scanfiler -c samples/config.yaml apply --proposals samples/proposals.jsonl +# results land in samples/library/ (gitignored); undo with: +scanfiler -c samples/config.yaml undo --last +``` + +The drawing has no text, so it exercises the low-confidence → `_Unsorted` path. +Regenerate the samples any time with `python samples/generate_samples.py`. + ## Key config | Key | Meaning | @@ -90,4 +109,25 @@ macOS/Windows: use `scanfiler loop` under launchd / Task Scheduler. ```bash pytest # stubs the AI client and generates sample PDFs/images; nothing external needed +ruff check . # lint ``` + +`pytest` enforces a coverage floor (`--cov-fail-under=90` in `pyproject.toml`). + +## Contributing & releases + +Changes land via **feature branch → pull request → merge into `main`**, not direct +commits to `main`. + +```bash +git checkout -b my-change +ruff check . && pytest +git push -u origin my-change +gh pr create --base main --fill +``` + +CI (`.github/workflows/ci.yml`) runs the gates on every PR and push: **ruff lint**, +**pytest + coverage gate**, and a **package build**, across Python 3.11/3.12/3.13. +On a push to `main` that passes all gates, the release job auto-increments the patch +version, tags it (`vX.Y.Z`), and publishes a GitHub Release with generated notes — so +direct commits to `main` would make those notes noisy; use PRs. diff --git a/pyproject.toml b/pyproject.toml index 3b56432..174859c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,9 @@ dependencies = [ [project.optional-dependencies] dev = [ "pytest>=8.0", + "pytest-cov>=5.0", + "ruff>=0.6", + "build>=1.2", ] [project.scripts] @@ -29,3 +32,23 @@ include = ["scanfiler*"] [tool.pytest.ini_options] testpaths = ["tests"] +addopts = "--cov=scanfiler --cov-report=term-missing --cov-fail-under=90" + +[tool.coverage.run] +omit = [ + "scanfiler/__main__.py", # thin `python -m scanfiler` shim +] + +[tool.coverage.report] +exclude_also = [ + "if __name__ == .__main__.:", + "raise SystemExit", +] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B"] +ignore = ["B904"] # allow `raise ... ` without `from` in retry/validation paths diff --git a/samples/config.yaml b/samples/config.yaml new file mode 100644 index 0000000..363ef14 --- /dev/null +++ b/samples/config.yaml @@ -0,0 +1,47 @@ +# Ready-to-run demo config for the committed sample documents. +# From the repo root, with a vision model served at ai.base_url: +# +# scanfiler -c samples/config.yaml plan --proposals samples/proposals.jsonl +# # review samples/proposals.jsonl, then: +# scanfiler -c samples/config.yaml apply --proposals samples/proposals.jsonl +# +# Outputs (samples/library, samples/state, samples/logs, proposals) are gitignored. + +paths: + inbox_dir: samples/inbox + library_dir: samples/library + unsorted_subdir: _Unsorted + +ai: + base_url: http://localhost:8080/v1 # point at your llama-server / mlx-vlm / cloud + api_key: ${AI_API_KEY} + model: local-vlm + constrained_output: true + +extraction: + pdf_max_pages: 2 + send_mode: auto + +selection: + process_pattern: '^(SCAN|PIC|IMG)[\W_]*\d+' + min_mtime_age_s: 0 # samples aren't being synced; process immediately + +naming: + date_prefix: true + allow_new_subdirs: true + +categorization: + confidence_threshold: 0.6 + +apply: + mode: review + action: copy + +logging: + audit_file: samples/logs/audit.jsonl + ledger_db: samples/state/ledger.sqlite + +prompt: + context: > + Demo set of personal scanned documents: an auto-service receipt, an electrician's + invoice, and a child's crayon drawing. diff --git a/samples/generate_samples.py b/samples/generate_samples.py new file mode 100644 index 0000000..c465adb --- /dev/null +++ b/samples/generate_samples.py @@ -0,0 +1,82 @@ +"""Regenerate the sample input documents in samples/inbox/. + +These committed samples let anyone try scanfiler locally without supplying their own +documents. Run from the repo root: + + python samples/generate_samples.py + +Then point scanfiler at them: + + scanfiler -c samples/config.yaml plan --proposals samples/proposals.jsonl +""" + +from __future__ import annotations + +from pathlib import Path + +INBOX = Path(__file__).resolve().parent / "inbox" + + +def _receipt_pdf(path: Path) -> None: + import fitz + + doc = fitz.open() + page = doc.new_page() + lines = [ + "RIVERSIDE AUTO SERVICE", + "123 Main St, Springfield", + "", + "RECEIPT # 4471", + "Date: 2025-06-15", + "", + "Kia Telluride - Brake pad replacement (front) $189.00", + "Synthetic oil change $ 64.00", + "Shop supplies $ 12.50", + "", + "TOTAL $265.50", + "Paid: VISA ****1234", + "Thank you for your business!", + ] + page.insert_text((72, 90), "\n".join(lines), fontsize=11) + doc.save(str(path)) + doc.close() + + +def _invoice_docx(path: Path) -> None: + import docx + + d = docx.Document() + d.add_heading("INVOICE", level=1) + d.add_paragraph("Bright Spark Electric LLC") + d.add_paragraph("Invoice #2025-0098 Date: 2025-05-02") + d.add_paragraph("Bill to: Jordan Avery") + t = d.add_table(rows=1, cols=2) + t.rows[0].cells[0].text = "Panel upgrade to 200A" + t.rows[0].cells[1].text = "$1,450.00" + d.add_paragraph("Amount due: $1,450.00 - Net 30") + d.save(str(path)) + + +def _drawing_image(path: Path) -> None: + from PIL import Image, ImageDraw + + im = Image.new("RGB", (600, 400), (255, 252, 240)) + draw = ImageDraw.Draw(im) + # A child's crayon-style house + sun, no text -> exercises the VLM / low-confidence path. + draw.rectangle([180, 200, 380, 340], outline=(60, 90, 200), width=6) + draw.polygon([(165, 200), (280, 110), (395, 200)], outline=(200, 60, 60), width=6) + draw.rectangle([250, 270, 310, 340], outline=(60, 150, 60), width=5) + draw.ellipse([470, 40, 560, 130], outline=(240, 190, 40), width=6) + im.save(str(path)) + + +def main() -> None: + INBOX.mkdir(parents=True, exist_ok=True) + _receipt_pdf(INBOX / "SCAN00001.pdf") + _invoice_docx(INBOX / "SCAN00002.docx") + _drawing_image(INBOX / "PIC00001.png") + print(f"Wrote samples to {INBOX}") + + +if __name__ == "__main__": + main() diff --git a/samples/inbox/PIC00001.png b/samples/inbox/PIC00001.png new file mode 100644 index 0000000..16f17e0 Binary files /dev/null and b/samples/inbox/PIC00001.png differ diff --git a/samples/inbox/SCAN00001.pdf b/samples/inbox/SCAN00001.pdf new file mode 100644 index 0000000..e7fb536 Binary files /dev/null and b/samples/inbox/SCAN00001.pdf differ diff --git a/samples/inbox/SCAN00002.docx b/samples/inbox/SCAN00002.docx new file mode 100644 index 0000000..8e303b3 Binary files /dev/null and b/samples/inbox/SCAN00002.docx differ diff --git a/scanfiler/ai/schema.py b/scanfiler/ai/schema.py index f0f32ca..74883f3 100644 --- a/scanfiler/ai/schema.py +++ b/scanfiler/ai/schema.py @@ -8,8 +8,6 @@ from __future__ import annotations -from typing import Optional - from pydantic import BaseModel, Field @@ -17,10 +15,12 @@ class Decision(BaseModel): """The model's proposal for a single file.""" filename: str = Field(description="Base name, NO extension; the tool re-adds the original") - subdir: str = Field(description="Target subfolder; one of the provided list unless is_new_subdir") + subdir: str = Field( + description="Target subfolder; one of the provided list unless is_new_subdir" + ) is_new_subdir: bool = False doc_type: str = "" - date: Optional[str] = None # ISO 'YYYY' / 'YYYY-MM' / 'YYYY-MM-DD'; null if unknown + date: str | None = None # ISO 'YYYY' / 'YYYY-MM' / 'YYYY-MM-DD'; null if unknown summary: str = "" tags: list[str] = Field(default_factory=list) confidence: float = 0.0 diff --git a/scanfiler/apply.py b/scanfiler/apply.py index 6985976..100923b 100644 --- a/scanfiler/apply.py +++ b/scanfiler/apply.py @@ -145,7 +145,8 @@ def undo(cfg: Config, *, run_id: str | None = None, last: bool = False) -> int: if not audit_file.is_file(): return 0 - records = [json.loads(line) for line in audit_file.read_text(encoding="utf-8").splitlines() if line.strip()] + lines = audit_file.read_text(encoding="utf-8").splitlines() + records = [json.loads(line) for line in lines if line.strip()] moves = [r for r in records if r.get("action") in ("copy", "move")] if not moves: return 0 diff --git a/scanfiler/cli.py b/scanfiler/cli.py index 16f282a..aa3dffc 100644 --- a/scanfiler/cli.py +++ b/scanfiler/cli.py @@ -163,16 +163,24 @@ def add_dry(sp): sp.set_defaults(func=cmd_init) sp = sub.add_parser("plan", help="extract + AI, write proposals (no moves)") - add_proposals(sp); add_dry(sp); sp.set_defaults(func=cmd_plan) + add_proposals(sp) + add_dry(sp) + sp.set_defaults(func=cmd_plan) sp = sub.add_parser("apply", help="execute proposals into the library") - add_proposals(sp); add_dry(sp); sp.set_defaults(func=cmd_apply) + add_proposals(sp) + add_dry(sp) + sp.set_defaults(func=cmd_apply) sp = sub.add_parser("run", help="one full cycle (plan, then apply if mode=auto)") - add_proposals(sp); add_dry(sp); sp.set_defaults(func=cmd_run) + add_proposals(sp) + add_dry(sp) + sp.set_defaults(func=cmd_run) sp = sub.add_parser("loop", help="daemon: run a cycle every polling_minutes") - add_proposals(sp); add_dry(sp); sp.set_defaults(func=cmd_loop) + add_proposals(sp) + add_dry(sp) + sp.set_defaults(func=cmd_loop) sp = sub.add_parser("undo", help="reverse a run from the audit log") sp.add_argument("--run", help="run id to undo") diff --git a/scanfiler/ledger.py b/scanfiler/ledger.py index 682e510..b150ba3 100644 --- a/scanfiler/ledger.py +++ b/scanfiler/ledger.py @@ -12,10 +12,10 @@ import json import sqlite3 import time +from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path -from typing import Iterator, Optional # Lifecycle of a file in the ledger. STATUS_PENDING = "pending" # seen, not yet decided @@ -43,8 +43,8 @@ class LedgerEntry: status: str decision: dict = field(default_factory=dict) metadata: dict = field(default_factory=dict) - new_path: Optional[str] = None - error: Optional[str] = None + new_path: str | None = None + error: str | None = None created_at: float = 0.0 updated_at: float = 0.0 @@ -77,13 +77,13 @@ def __init__(self, db_path: str | Path): def close(self) -> None: self._conn.close() - def __enter__(self) -> "Ledger": + def __enter__(self) -> Ledger: return self def __exit__(self, *exc) -> None: self.close() - def get(self, file_hash: str) -> Optional[LedgerEntry]: + def get(self, file_hash: str) -> LedgerEntry | None: row = self._conn.execute( "SELECT * FROM files WHERE file_hash = ?", (file_hash,) ).fetchone() diff --git a/scanfiler/lock.py b/scanfiler/lock.py index 6f55508..0fb84df 100644 --- a/scanfiler/lock.py +++ b/scanfiler/lock.py @@ -8,9 +8,9 @@ import os import time +from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path -from typing import Iterator _STALE_AGE_S = 6 * 60 * 60 # reclaim locks older than this (crash safety net) diff --git a/scanfiler/proposals.py b/scanfiler/proposals.py index d971fe7..4389983 100644 --- a/scanfiler/proposals.py +++ b/scanfiler/proposals.py @@ -8,9 +8,9 @@ from __future__ import annotations import json +from collections.abc import Iterator from dataclasses import asdict, dataclass, field from pathlib import Path -from typing import Iterator @dataclass diff --git a/tests/conftest.py b/tests/conftest.py index 8c1e5e1..1c5c160 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,7 +41,10 @@ def workspace(tmp_path: Path) -> dict: inbox.mkdir() library.mkdir() - _make_pdf(inbox / "SCAN0001.pdf", "INVOICE\nAcme Plumbing\nTotal due: $240.00\nDate: 2025-06-15") + _make_pdf( + inbox / "SCAN0001.pdf", + "INVOICE\nAcme Plumbing\nTotal due: $240.00\nDate: 2025-06-15", + ) _make_docx(inbox / "SCAN0002.docx", "Medical record: annual checkup notes for patient.") _make_image(inbox / "PIC0001.jpg") # Already-named file that should be skipped unless process_all is on. @@ -95,3 +98,19 @@ def decide(self, system_prompt, user_content, existing_subdirs, allow_new) -> De @pytest.fixture def stub_client() -> StubClient: return StubClient() + + +@pytest.fixture +def make_pdf(): + """Factory for writing a small text PDF (avoids cross-module test imports).""" + return _make_pdf + + +@pytest.fixture +def config_file(workspace, config) -> Path: + """Write the test config to a YAML file on disk for CLI-level tests.""" + import yaml + + path = workspace["root"] / "config.yaml" + path.write_text(yaml.safe_dump(config.model_dump(mode="json")), encoding="utf-8") + return path diff --git a/tests/test_apply.py b/tests/test_apply.py new file mode 100644 index 0000000..e176c17 --- /dev/null +++ b/tests/test_apply.py @@ -0,0 +1,92 @@ +"""Edge cases for apply/undo beyond the happy path in test_pipeline.""" + +from __future__ import annotations + +import json + +from scanfiler.apply import apply_proposals, undo +from scanfiler.ledger import Ledger +from scanfiler.proposals import Proposal + + +def _proposal(workspace, **over): + src = workspace["inbox"] / "SCAN0001.pdf" + base = dict(file_hash="h1", original_path=str(src), subdir="Receipts", + new_filename="Receipt.pdf", confidence=0.9) + base.update(over) + return Proposal(**base) + + +def test_move_action_removes_original(config, workspace): + config.apply.action = "move" + src = workspace["inbox"] / "SCAN0001.pdf" + with Ledger(config.logging.ledger_db) as ledger: + apply_proposals(config, [_proposal(workspace)], ledger) + assert not src.exists() # moved out + assert (config.paths.library_dir / "Receipts" / "Receipt.pdf").is_file() + + +def test_move_undo_restores_to_inbox(config, workspace): + config.apply.action = "move" + src = workspace["inbox"] / "SCAN0001.pdf" + with Ledger(config.logging.ledger_db) as ledger: + apply_proposals(config, [_proposal(workspace)], ledger) + assert not src.exists() + restored = undo(config, last=True) + assert restored == 1 + assert src.exists() # moved back + + +def test_dry_run_touches_nothing(config, workspace): + with Ledger(config.logging.ledger_db) as ledger: + result = apply_proposals(config, [_proposal(workspace)], ledger, dry_run=True) + assert result.applied == 1 + assert not (config.paths.library_dir / "Receipts").exists() + assert not config.logging.audit_file.exists() + + +def test_missing_source_is_skipped(config): + p = Proposal(file_hash="h", original_path="/does/not/exist.pdf", + subdir="X", new_filename="Y.pdf") + with Ledger(config.logging.ledger_db) as ledger: + result = apply_proposals(config, [p], ledger) + assert result.skipped == 1 and result.applied == 0 + + +def test_collision_skip_policy(config, workspace): + config.apply.on_collision = "skip" + dest = config.paths.library_dir / "Receipts" / "Receipt.pdf" + dest.parent.mkdir(parents=True) + dest.write_text("existing", encoding="utf-8") + with Ledger(config.logging.ledger_db) as ledger: + result = apply_proposals(config, [_proposal(workspace)], ledger) + assert result.skipped == 1 + assert dest.read_text(encoding="utf-8") == "existing" # untouched + + +def test_error_is_recorded_not_raised(config, workspace, monkeypatch): + import scanfiler.apply as apply_mod + + def boom(*a, **k): + raise OSError("disk full") + + monkeypatch.setattr(apply_mod.shutil, "copy2", boom) + with Ledger(config.logging.ledger_db) as ledger: + result = apply_proposals(config, [_proposal(workspace)], ledger) + assert result.errors == 1 + records = [json.loads(x) for x in config.logging.audit_file.read_text().splitlines()] + assert any(r["action"] == "error" for r in records) + + +def test_undo_no_audit_file_returns_zero(config): + assert undo(config, last=True) == 0 + + +def test_sidecar_written_with_metadata(config, workspace): + p = _proposal(workspace, doc_type="receipt", tags=["x"], summary="hi", date="2025-06") + with Ledger(config.logging.ledger_db) as ledger: + apply_proposals(config, [p], ledger) + sidecar = config.paths.library_dir / "Receipts" / "Receipt.pdf.json" + data = json.loads(sidecar.read_text(encoding="utf-8")) + assert data["doc_type"] == "receipt" + assert data["source_hash"] == "h1" diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..7bd3f41 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,115 @@ +"""CLI-level tests: drive scanfiler.cli.main with a stubbed AI client.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import scanfiler.cli as cli + + +@pytest.fixture(autouse=True) +def _patch_client(monkeypatch, stub_client): + monkeypatch.setattr(cli, "_make_client", lambda cfg: stub_client) + + +def test_init_writes_and_refuses_overwrite(tmp_path, capsys): + cfgp = tmp_path / "config.yaml" + assert cli.main(["-c", str(cfgp), "init"]) == 0 + assert cfgp.is_file() + # second time refuses + assert cli.main(["-c", str(cfgp), "init"]) == 1 + assert "not overwriting" in capsys.readouterr().out + + +def test_plan_then_apply_then_status_then_undo(config_file, workspace, capsys): + c = str(config_file) + proposals = str(workspace["root"] / "proposals.jsonl") + + assert cli.main(["-c", c, "plan", "--proposals", proposals]) == 0 + assert Path(proposals).is_file() + assert "proposed=" in capsys.readouterr().out + + assert cli.main(["-c", c, "apply", "--proposals", proposals]) == 0 + assert "applied=" in capsys.readouterr().out + + assert cli.main(["-c", c, "status"]) == 0 + assert "applied" in capsys.readouterr().out + + assert cli.main(["-c", c, "undo", "--last"]) == 0 + assert "restored" in capsys.readouterr().out + + +def test_plan_dry_run_writes_nothing(config_file, workspace, capsys): + proposals = workspace["root"] / "proposals.jsonl" + cli.main(["-c", str(config_file), "plan", "--proposals", str(proposals), "--dry-run"]) + assert not proposals.exists() + assert "dry-run" in capsys.readouterr().out + + +def test_run_review_mode_writes_proposals(config_file, workspace, capsys): + proposals = workspace["root"] / "proposals.jsonl" + assert cli.main(["-c", str(config_file), "run", "--proposals", str(proposals)]) == 0 + assert proposals.is_file() + + +def test_run_auto_mode_applies(config_file, config, workspace, monkeypatch, capsys): + # Flip mode=auto by rewriting the config file. + import yaml + + data = yaml.safe_load(config_file.read_text(encoding="utf-8")) + data["apply"]["mode"] = "auto" + config_file.write_text(yaml.safe_dump(data), encoding="utf-8") + + assert cli.main(["-c", str(config_file), "run"]) == 0 + assert "auto-apply" in capsys.readouterr().out + # something landed in the library + assert any(config.paths.library_dir.rglob("*.pdf")) + + +def test_run_skips_when_locked(config_file, config, capsys): + from scanfiler.lock import file_lock + + lock_path = Path(config.logging.ledger_db).with_suffix(".lock") + with file_lock(lock_path): # hold the lock; the run should bail out cleanly + assert cli.main(["-c", str(config_file), "run"]) == 0 + assert "skip:" in capsys.readouterr().out + + +def test_status_empty_ledger(config_file, capsys): + assert cli.main(["-c", str(config_file), "status"]) == 0 + assert "ledger empty" in capsys.readouterr().out + + +def test_undo_nonexistent_run(config_file, capsys): + assert cli.main(["-c", str(config_file), "undo", "--run", "deadbeef"]) == 0 + assert "restored 0" in capsys.readouterr().out + + +def test_loop_stops_on_keyboard_interrupt(config_file, monkeypatch, capsys): + def stop(*a, **k): + raise KeyboardInterrupt + + monkeypatch.setattr(cli, "_run_once", stop) + assert cli.main(["-c", str(config_file), "loop"]) == 0 + assert "stopped" in capsys.readouterr().out + + +def test_loop_survives_a_bad_cycle(config_file, monkeypatch, capsys): + calls = {"n": 0} + + def flaky(cfg, args): + calls["n"] += 1 + raise RuntimeError("transient cycle failure") + + # First sleep ends the loop so the test doesn't hang; the cycle error must be caught. + def fake_sleep(_): + raise KeyboardInterrupt + + monkeypatch.setattr(cli, "_run_once", flaky) + monkeypatch.setattr(cli.time, "sleep", fake_sleep) + with pytest.raises(KeyboardInterrupt): + cli.main(["-c", str(config_file), "loop"]) + assert calls["n"] == 1 + assert "cycle error" in capsys.readouterr().err diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..76609c9 --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,102 @@ +"""AIClient tests with a fake httpx transport (no network).""" + +from __future__ import annotations + +import json + +import pytest + +from scanfiler.ai.client import OpenAICompatClient, make_client +from scanfiler.config import AIConfig + + +class _FakeResponse: + def __init__(self, payload, status_ok=True): + self._payload = payload + self._ok = status_ok + + def raise_for_status(self): + if not self._ok: + raise RuntimeError("HTTP 500") + + def json(self): + return self._payload + + +class _FakeClient: + """Stands in for httpx.Client; scripted per-call responses.""" + + script: list = [] + posted: list = [] + + def __init__(self, *a, **k): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def post(self, url, json=None, headers=None): + _FakeClient.posted.append({"url": url, "json": json, "headers": headers}) + item = _FakeClient.script.pop(0) + if isinstance(item, Exception): + raise item + return item + + +def _decision_payload(**over): + base = {"filename": "Doc", "subdir": "Misc", "confidence": 0.7} + base.update(over) + return {"choices": [{"message": {"content": json.dumps(base)}}]} + + +@pytest.fixture(autouse=True) +def _patch_httpx(monkeypatch): + import httpx + + _FakeClient.script = [] + _FakeClient.posted = [] + monkeypatch.setattr(httpx, "Client", _FakeClient) + + +def _client(**over): + cfg = AIConfig(max_retries=3, request_timeout_s=1, **over) + return OpenAICompatClient(cfg) + + +def test_decide_success_parses_decision(): + _FakeClient.script = [_FakeResponse(_decision_payload(filename="Invoice"))] + d = _client().decide("sys", [{"type": "text", "text": "x"}], ["Misc"], True) + assert d.filename == "Invoice" + assert d.confidence == 0.7 + + +def test_decide_retries_then_succeeds(): + _FakeClient.script = [RuntimeError("boom"), _FakeResponse(_decision_payload())] + d = _client().decide("sys", [], [], True) + assert d.filename == "Doc" + assert len(_FakeClient.posted) == 2 # one failed, one succeeded + + +def test_decide_exhausts_retries_and_raises(): + _FakeClient.script = [RuntimeError("a"), RuntimeError("b"), RuntimeError("c")] + with pytest.raises(RuntimeError, match="failed after 3 attempts"): + _client().decide("sys", [], [], True) + + +def test_constrained_output_adds_response_format(): + _FakeClient.script = [_FakeResponse(_decision_payload())] + _client(constrained_output=True).decide("sys", [], ["A", "B"], False) + assert "response_format" in _FakeClient.posted[0]["json"] + + +def test_api_key_sets_auth_header(): + _FakeClient.script = [_FakeResponse(_decision_payload())] + _client(api_key="tok").decide("sys", [], [], True) + assert _FakeClient.posted[0]["headers"]["Authorization"] == "Bearer tok" + + +def test_make_client_returns_openai_compat(): + assert isinstance(make_client(AIConfig()), OpenAICompatClient) diff --git a/tests/test_config.py b/tests/test_config.py index fe02a28..e49bf56 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,3 @@ -import os from scanfiler.config import load_config diff --git a/tests/test_extract.py b/tests/test_extract.py new file mode 100644 index 0000000..61abe78 --- /dev/null +++ b/tests/test_extract.py @@ -0,0 +1,85 @@ +from pathlib import Path + +from scanfiler.config import ExtractionConfig +from scanfiler.extract import classify, extract + + +def test_classify(): + assert classify(Path("a.pdf")) == "pdf" + assert classify(Path("a.DOCX")) == "docx" + assert classify(Path("a.JPG")) == "image" + assert classify(Path("a.xyz")) == "unknown" + + +def test_unknown_type_returns_error(tmp_path): + f = tmp_path / "weird.xyz" + f.write_text("hello", encoding="utf-8") + r = extract(f, ExtractionConfig()) + assert r.kind == "unknown" + assert r.error is not None + assert r.has_content is False + + +def test_corrupt_pdf_surfaces_error(tmp_path): + f = tmp_path / "broken.pdf" + f.write_bytes(b"%PDF-1.4 not really a pdf") + r = extract(f, ExtractionConfig()) + assert r.error is not None # exception captured, not raised + + +def test_pdf_text_mode_carries_no_images(tmp_path): + import fitz + + f = tmp_path / "doc.pdf" + doc = fitz.open() + doc.new_page().insert_text((72, 72), "Plenty of words " * 30) + doc.save(str(f)) + doc.close() + + r = extract(f, ExtractionConfig(send_mode="text")) + assert r.text.strip() + assert r.images == [] + + +def test_pdf_vision_mode_renders_images(tmp_path): + import fitz + + f = tmp_path / "doc.pdf" + doc = fitz.open() + doc.new_page().insert_text((72, 72), "x") + doc.save(str(f)) + doc.close() + + r = extract(f, ExtractionConfig(send_mode="vision", pdf_max_pages=1)) + assert len(r.images) == 1 + + +def test_image_downscaled_and_pngified(tmp_path): + from PIL import Image + + f = tmp_path / "big.png" + Image.new("RGB", (4000, 100), (1, 2, 3)).save(str(f)) + r = extract(f, ExtractionConfig()) + assert r.kind == "image" + assert len(r.images) == 1 + # re-open the produced PNG; longest edge must be capped + import io + + with Image.open(io.BytesIO(r.images[0])) as im: + assert max(im.size) <= 1600 + + +def test_docx_extracts_paragraphs_and_tables(tmp_path): + import docx + + f = tmp_path / "d.docx" + d = docx.Document() + d.add_paragraph("Hello world") + t = d.add_table(rows=1, cols=2) + t.rows[0].cells[0].text = "Key" + t.rows[0].cells[1].text = "Value" + d.save(str(f)) + + r = extract(f, ExtractionConfig()) + assert "Hello world" in r.text + assert "Key | Value" in r.text diff --git a/tests/test_ledger.py b/tests/test_ledger.py new file mode 100644 index 0000000..e145797 --- /dev/null +++ b/tests/test_ledger.py @@ -0,0 +1,57 @@ +from scanfiler.ledger import ( + STATUS_APPLIED, + STATUS_ERROR, + STATUS_PENDING, + LedgerEntry, + hash_file, + open_ledger, +) + + +def test_hash_file_stable(tmp_path): + f = tmp_path / "a.bin" + f.write_bytes(b"hello world") + assert hash_file(f) == hash_file(f) + g = tmp_path / "b.bin" + g.write_bytes(b"hello world!") + assert hash_file(f) != hash_file(g) + + +def test_upsert_and_get(tmp_path): + with open_ledger(tmp_path / "l.sqlite") as ledger: + ledger.upsert(LedgerEntry(file_hash="h", original_name="x.pdf", + status=STATUS_PENDING)) + e = ledger.get("h") + assert e is not None and e.status == STATUS_PENDING + assert e.created_at > 0 and e.updated_at >= e.created_at + + +def test_upsert_updates_in_place_keeping_created(tmp_path): + with open_ledger(tmp_path / "l.sqlite") as ledger: + ledger.upsert(LedgerEntry(file_hash="h", original_name="x", status=STATUS_PENDING)) + created = ledger.get("h").created_at + ledger.upsert(LedgerEntry(file_hash="h", original_name="x", status=STATUS_APPLIED, + metadata={"k": "v"})) + e = ledger.get("h") + assert e.status == STATUS_APPLIED + assert e.metadata == {"k": "v"} + assert e.created_at == created + + +def test_seen_only_for_decided_states(tmp_path): + with open_ledger(tmp_path / "l.sqlite") as ledger: + ledger.upsert(LedgerEntry(file_hash="p", original_name="x", status=STATUS_PENDING)) + ledger.upsert(LedgerEntry(file_hash="e", original_name="y", status=STATUS_ERROR)) + ledger.upsert(LedgerEntry(file_hash="a", original_name="z", status=STATUS_APPLIED)) + assert ledger.seen("p") is False # pending -> retry + assert ledger.seen("e") is False # error -> retry + assert ledger.seen("a") is True + assert ledger.seen("missing") is False + + +def test_counts(tmp_path): + with open_ledger(tmp_path / "l.sqlite") as ledger: + ledger.upsert(LedgerEntry(file_hash="a", original_name="x", status=STATUS_APPLIED)) + ledger.upsert(LedgerEntry(file_hash="b", original_name="y", status=STATUS_APPLIED)) + ledger.upsert(LedgerEntry(file_hash="c", original_name="z", status=STATUS_ERROR)) + assert ledger.counts() == {STATUS_APPLIED: 2, STATUS_ERROR: 1} diff --git a/tests/test_lock.py b/tests/test_lock.py new file mode 100644 index 0000000..8f2ced0 --- /dev/null +++ b/tests/test_lock.py @@ -0,0 +1,54 @@ +import os +import sys +import time + +import pytest + +from scanfiler.lock import _STALE_AGE_S, LockHeld, file_lock + + +def test_lock_acquire_release(tmp_path): + lp = tmp_path / "x.lock" + with file_lock(lp): + assert lp.exists() + assert not lp.exists() # released on exit + + +def test_lock_held_raises(tmp_path): + lp = tmp_path / "x.lock" + with file_lock(lp): + with pytest.raises(LockHeld): + with file_lock(lp): + pass + + +def test_stale_lock_reclaimed_by_age(tmp_path): + # An old lock is reclaimed by age on every platform (crash safety net). + lp = tmp_path / "x.lock" + lp.write_text(f"{os.getpid()} 0", encoding="utf-8") + old = time.time() - _STALE_AGE_S - 60 + os.utime(lp, (old, old)) + with file_lock(lp): + assert lp.exists() + + +@pytest.mark.skipif(sys.platform == "win32", reason="PID liveness check is POSIX-only") +def test_stale_lock_reclaimed_by_dead_pid(tmp_path): + lp = tmp_path / "x.lock" + lp.write_text("999999999 0", encoding="utf-8") # definitely-dead PID + with file_lock(lp): + assert lp.exists() + + +def test_garbage_lock_reclaimed(tmp_path): + lp = tmp_path / "x.lock" + lp.write_text("not-a-pid", encoding="utf-8") + with file_lock(lp): + assert lp.exists() + + +def test_lock_writes_pid(tmp_path): + lp = tmp_path / "x.lock" + with file_lock(lp): + content = lp.read_text(encoding="utf-8") + assert content.split()[0] == str(os.getpid()) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index c7d63c7..5f15414 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,7 +1,7 @@ from pathlib import Path from scanfiler.apply import apply_proposals, undo -from scanfiler.ledger import STATUS_APPLIED, Ledger, hash_file +from scanfiler.ledger import Ledger from scanfiler.pipeline import plan @@ -71,11 +71,9 @@ def test_undo_restores(config, stub_client): ledger.close() -def test_collision_resolution_same_batch(config, stub_client, workspace): +def test_collision_resolution_same_batch(config, stub_client, workspace, make_pdf): # Two files that both extract as "invoice" -> same desired name -> must dedupe. - from tests.conftest import _make_pdf - - _make_pdf(workspace["inbox"] / "SCAN0009.pdf", "INVOICE second one 2025-06") + make_pdf(workspace["inbox"] / "SCAN0009.pdf", "INVOICE second one 2025-06") ledger, proposals, _ = _plan(config, stub_client) invoice_names = [p.new_filename for p in proposals if p.subdir == "Invoices"] assert len(invoice_names) == len(set(invoice_names)) # all unique diff --git a/tests/test_pipeline_edges.py b/tests/test_pipeline_edges.py new file mode 100644 index 0000000..bf36127 --- /dev/null +++ b/tests/test_pipeline_edges.py @@ -0,0 +1,72 @@ +"""Selection and routing edge cases in the pipeline.""" + +from __future__ import annotations + +from pathlib import Path + +from scanfiler.ai.schema import Decision +from scanfiler.ledger import STATUS_ERROR, Ledger +from scanfiler.pipeline import iter_inbox, plan + + +class _FixedClient: + def __init__(self, decision): + self.decision = decision + + def decide(self, *a, **k): + return self.decision + + +def test_process_all_includes_named_files(config, stub_client, workspace): + config.selection.process_all = True + with Ledger(config.logging.ledger_db) as ledger: + proposals, _ = plan(config, stub_client, ledger) + names = {Path(p.original_path).name for p in proposals} + assert "TaxReturn2024.pdf" in names # normally excluded by process_pattern + + +def test_blocked_new_subdir_routes_to_unsorted(config, workspace): + config.naming.allow_new_subdirs = False + client = _FixedClient( + Decision(filename="Doc", subdir="BrandNew", is_new_subdir=True, confidence=0.99) + ) + with Ledger(config.logging.ledger_db) as ledger: + proposals, stats = plan(config, client, ledger) + assert all(p.subdir == config.paths.unsorted_subdir for p in proposals) + assert stats.unsorted == len(proposals) + + +def test_extraction_failure_marks_error(config, stub_client, workspace): + # An empty PDF extension with no content -> extraction yields no content -> error status. + bad = workspace["inbox"] / "SCAN0003.pdf" + bad.write_bytes(b"%PDF-1.4 garbage") + with Ledger(config.logging.ledger_db) as ledger: + plan(config, stub_client, ledger) + entries = ledger.by_status(STATUS_ERROR) + assert any(e.original_name == "SCAN0003.pdf" for e in entries) + + +def test_ai_exception_marks_error(config, workspace): + class _Boom: + def decide(self, *a, **k): + raise RuntimeError("model down") + + with Ledger(config.logging.ledger_db) as ledger: + proposals, stats = plan(config, _Boom(), ledger) + errors = ledger.by_status(STATUS_ERROR) + assert proposals == [] + assert stats.errors >= 1 + assert any("ai:" in (e.error or "") for e in errors) + + +def test_mtime_guard_excludes_fresh_files(config, workspace): + config.selection.min_mtime_age_s = 3600 # everything is "too fresh" + assert list(iter_inbox(config)) == [] + + +def test_ignore_globs_exclude_hidden_and_partial(config, workspace): + (workspace["inbox"] / ".hidden.pdf").write_bytes(b"%PDF-1.4") + (workspace["inbox"] / "SCAN0007.pdf.partial").write_bytes(b"x") + selected = {p.name for p in iter_inbox(config)} + assert ".hidden.pdf" not in selected + assert "SCAN0007.pdf.partial" not in selected diff --git a/tests/test_proposals.py b/tests/test_proposals.py new file mode 100644 index 0000000..4595264 --- /dev/null +++ b/tests/test_proposals.py @@ -0,0 +1,33 @@ +import pytest + +from scanfiler.proposals import Proposal, read_proposals, write_proposals + + +def _p(**over): + base = dict(file_hash="h1", original_path="/in/SCAN0001.pdf", subdir="Receipts", + new_filename="Receipt.pdf") + base.update(over) + return Proposal(**base) + + +def test_roundtrip(tmp_path): + path = tmp_path / "p.jsonl" + items = [_p(), _p(file_hash="h2", tags=["a", "b"], confidence=0.9)] + write_proposals(path, items) + back = list(read_proposals(path)) + assert back == items + + +def test_skips_blank_and_comment_lines(tmp_path): + path = tmp_path / "p.jsonl" + write_proposals(path, [_p()]) + with open(path, "a", encoding="utf-8") as f: + f.write("\n# a hand comment\n") + assert len(list(read_proposals(path))) == 1 + + +def test_invalid_json_raises_with_line_number(tmp_path): + path = tmp_path / "p.jsonl" + path.write_text('{"file_hash": "h1"\n', encoding="utf-8") # truncated JSON + with pytest.raises(ValueError, match=":1:"): + list(read_proposals(path)) diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 0000000..fe97a29 --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,26 @@ +from scanfiler.ai.schema import Decision, build_response_format + + +def test_enum_constrained_when_new_disallowed(): + rf = build_response_format(["A", "B"], allow_new=False) + subdir = rf["json_schema"]["schema"]["properties"]["subdir"] + assert subdir == {"type": "string", "enum": ["A", "B"]} + + +def test_open_string_when_new_allowed(): + rf = build_response_format(["A", "B"], allow_new=True) + subdir = rf["json_schema"]["schema"]["properties"]["subdir"] + assert subdir == {"type": "string"} + assert rf["json_schema"]["strict"] is True + + +def test_open_string_when_no_existing_subdirs(): + rf = build_response_format([], allow_new=False) + assert rf["json_schema"]["schema"]["properties"]["subdir"] == {"type": "string"} + + +def test_decision_defaults(): + d = Decision(filename="X", subdir="Y", confidence=0.5) + assert d.is_new_subdir is False + assert d.tags == [] + assert d.date is None