Add CI/CD pipeline, expand test coverage to 96%, ship sample documents

.github/workflows/ci.yml

@@ -0,0 +1,87 @@
+name: CI
+
+# Gates run on every PR and every push to main. A release is cut only when a push
+# to main passes all gates.
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+# Don't let overlapping runs race on the release/tag step. PR runs supersede each
+# other; runs on main are never cancelled mid-release.
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+permissions:
+  contents: read
+
+jobs:
+  gate:
+    name: Gate (Python ${{ matrix.python }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ['3.11', '3.12', '3.13']
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Lint (ruff)
+        run: ruff check .
+
+      - name: Test (pytest + coverage gate)
+        run: pytest
+
+      - name: Build (sdist + wheel)
+        run: python -m build
+
+  release:
+    name: Release
+    needs: [gate]
+    # Only on direct pushes to main (not PRs, not forks).
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # create tags and releases
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # need full tag history to compute the next version
+
+      - name: Determine next version
+        id: ver
+        run: |
+          latest=$(git tag -l 'v*.*.*' --sort=-v:refname | head -n1)
+          if [ -z "$latest" ]; then
+            next="v0.1.0"
+          else
+            ver=${latest#v}
+            IFS='.' read -r major minor patch <<< "$ver"
+            next="v${major}.${minor}.$((patch + 1))"
+          fi
+          echo "Next release: $next (previous: ${latest:-none})"
+          echo "next=$next" >> "$GITHUB_OUTPUT"
+
+      - name: Create tag and GitHub release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          next="${{ steps.ver.outputs.next }}"
+          git tag "$next"
+          git push origin "$next"
+          gh release create "$next" \
+            --title "$next" \
+            --target "${{ github.sha }}" \
+            --generate-notes

.gitignore

@@ -6,6 +6,13 @@ __pycache__/
 build/
 dist/
 
+# Test / coverage artifacts
+.coverage
+.coverage.*
+htmlcov/
+.pytest_cache/
+.ruff_cache/
+
 # Runtime state / secrets / user config
 config.yaml
 secrets.env
@@ -14,6 +21,13 @@ proposals.jsonl
 /state/
 *.lock
 
+# Sample demo: keep the input documents + demo config, ignore generated outputs.
+!samples/config.yaml
+/samples/library/
+/samples/state/
+/samples/logs/
+/samples/proposals.jsonl
+
 # OS
 .DS_Store
 Thumbs.db

README.md

@@ -1,5 +1,7 @@
 # scanfiler
 
+[![CI](https://github.com/ridaken/Scanfiler/actions/workflows/ci.yml/badge.svg)](https://github.com/ridaken/Scanfiler/actions/workflows/ci.yml)
+
 AI-powered renamer / reorganizer for a dump of scanned documents. It walks an inbox
 **one file at a time**, sends the first page(s) to an OpenAI-compatible vision-language
 model (built for [llama.cpp / llama-server](https://github.com/ggml-org/llama.cpp) and
@@ -53,6 +55,23 @@ scanfiler undo --last          # reverse the most recent apply run
 scanfiler <cmd> --dry-run      # decide + log, never touch disk
 ```
 
+## Try it on the bundled samples
+
+The repo ships three sample inputs in `samples/inbox/` (an auto-service receipt PDF,
+an electrician's invoice docx, and a child's crayon drawing PNG) and a ready-to-run
+`samples/config.yaml`. Point `ai.base_url` at a vision model, then:
+
+```bash
+scanfiler -c samples/config.yaml plan --proposals samples/proposals.jsonl
+# review samples/proposals.jsonl, then:
+scanfiler -c samples/config.yaml apply --proposals samples/proposals.jsonl
+# results land in samples/library/ (gitignored); undo with:
+scanfiler -c samples/config.yaml undo --last
+```
+
+The drawing has no text, so it exercises the low-confidence → `_Unsorted` path.
+Regenerate the samples any time with `python samples/generate_samples.py`.
+
 ## Key config
 
 | Key | Meaning |
@@ -90,4 +109,25 @@ macOS/Windows: use `scanfiler loop` under launchd / Task Scheduler.
 
 ```bash
 pytest          # stubs the AI client and generates sample PDFs/images; nothing external needed
+ruff check .    # lint
 ```
+
+`pytest` enforces a coverage floor (`--cov-fail-under=90` in `pyproject.toml`).
+
+## Contributing & releases
+
+Changes land via **feature branch → pull request → merge into `main`**, not direct
+commits to `main`.
+
+```bash
+git checkout -b my-change
+ruff check . && pytest
+git push -u origin my-change
+gh pr create --base main --fill
+```
+
+CI (`.github/workflows/ci.yml`) runs the gates on every PR and push: **ruff lint**,
+**pytest + coverage gate**, and a **package build**, across Python 3.11/3.12/3.13.
+On a push to `main` that passes all gates, the release job auto-increments the patch
+version, tags it (`vX.Y.Z`), and publishes a GitHub Release with generated notes — so
+direct commits to `main` would make those notes noisy; use PRs.

pyproject.toml

@@ -15,6 +15,9 @@ dependencies = [
 [project.optional-dependencies]
 dev = [
     "pytest>=8.0",
+    "pytest-cov>=5.0",
+    "ruff>=0.6",
+    "build>=1.2",
 ]
 
 [project.scripts]
@@ -29,3 +32,23 @@ include = ["scanfiler*"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+addopts = "--cov=scanfiler --cov-report=term-missing --cov-fail-under=90"
+
+[tool.coverage.run]
+omit = [
+    "scanfiler/__main__.py",   # thin `python -m scanfiler` shim
+]
+
+[tool.coverage.report]
+exclude_also = [
+    "if __name__ == .__main__.:",
+    "raise SystemExit",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]
+ignore = ["B904"]  # allow `raise ... ` without `from` in retry/validation paths

samples/config.yaml

@@ -0,0 +1,47 @@
+# Ready-to-run demo config for the committed sample documents.
+# From the repo root, with a vision model served at ai.base_url:
+#
+#   scanfiler -c samples/config.yaml plan --proposals samples/proposals.jsonl
+#   # review samples/proposals.jsonl, then:
+#   scanfiler -c samples/config.yaml apply --proposals samples/proposals.jsonl
+#
+# Outputs (samples/library, samples/state, samples/logs, proposals) are gitignored.
+
+paths:
+  inbox_dir: samples/inbox
+  library_dir: samples/library
+  unsorted_subdir: _Unsorted
+
+ai:
+  base_url: http://localhost:8080/v1    # point at your llama-server / mlx-vlm / cloud
+  api_key: ${AI_API_KEY}
+  model: local-vlm
+  constrained_output: true
+
+extraction:
+  pdf_max_pages: 2
+  send_mode: auto
+
+selection:
+  process_pattern: '^(SCAN|PIC|IMG)[\W_]*\d+'
+  min_mtime_age_s: 0                     # samples aren't being synced; process immediately
+
+naming:
+  date_prefix: true
+  allow_new_subdirs: true
+
+categorization:
+  confidence_threshold: 0.6
+
+apply:
+  mode: review
+  action: copy
+
+logging:
+  audit_file: samples/logs/audit.jsonl
+  ledger_db: samples/state/ledger.sqlite
+
+prompt:
+  context: >
+    Demo set of personal scanned documents: an auto-service receipt, an electrician's
+    invoice, and a child's crayon drawing.

samples/generate_samples.py

@@ -0,0 +1,82 @@
+"""Regenerate the sample input documents in samples/inbox/.
+
+These committed samples let anyone try scanfiler locally without supplying their own
+documents. Run from the repo root:
+
+    python samples/generate_samples.py
+
+Then point scanfiler at them:
+
+    scanfiler -c samples/config.yaml plan --proposals samples/proposals.jsonl
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+INBOX = Path(__file__).resolve().parent / "inbox"
+
+
+def _receipt_pdf(path: Path) -> None:
+    import fitz
+
+    doc = fitz.open()
+    page = doc.new_page()
+    lines = [
+        "RIVERSIDE AUTO SERVICE",
+        "123 Main St, Springfield",
+        "",
+        "RECEIPT  #  4471",
+        "Date: 2025-06-15",
+        "",
+        "Kia Telluride - Brake pad replacement (front)   $189.00",
+        "Synthetic oil change                            $ 64.00",
+        "Shop supplies                                   $ 12.50",
+        "",
+        "TOTAL                                           $265.50",
+        "Paid: VISA ****1234",
+        "Thank you for your business!",
+    ]
+    page.insert_text((72, 90), "\n".join(lines), fontsize=11)
+    doc.save(str(path))
+    doc.close()
+
+
+def _invoice_docx(path: Path) -> None:
+    import docx
+
+    d = docx.Document()
+    d.add_heading("INVOICE", level=1)
+    d.add_paragraph("Bright Spark Electric LLC")
+    d.add_paragraph("Invoice #2025-0098    Date: 2025-05-02")
+    d.add_paragraph("Bill to: Jordan Avery")
+    t = d.add_table(rows=1, cols=2)
+    t.rows[0].cells[0].text = "Panel upgrade to 200A"
+    t.rows[0].cells[1].text = "$1,450.00"
+    d.add_paragraph("Amount due: $1,450.00  -  Net 30")
+    d.save(str(path))
+
+
+def _drawing_image(path: Path) -> None:
+    from PIL import Image, ImageDraw
+
+    im = Image.new("RGB", (600, 400), (255, 252, 240))
+    draw = ImageDraw.Draw(im)
+    # A child's crayon-style house + sun, no text -> exercises the VLM / low-confidence path.
+    draw.rectangle([180, 200, 380, 340], outline=(60, 90, 200), width=6)
+    draw.polygon([(165, 200), (280, 110), (395, 200)], outline=(200, 60, 60), width=6)
+    draw.rectangle([250, 270, 310, 340], outline=(60, 150, 60), width=5)
+    draw.ellipse([470, 40, 560, 130], outline=(240, 190, 40), width=6)
+    im.save(str(path))
+
+
+def main() -> None:
+    INBOX.mkdir(parents=True, exist_ok=True)
+    _receipt_pdf(INBOX / "SCAN00001.pdf")
+    _invoice_docx(INBOX / "SCAN00002.docx")
+    _drawing_image(INBOX / "PIC00001.png")
+    print(f"Wrote samples to {INBOX}")
+
+
+if __name__ == "__main__":
+    main()

scanfiler/ai/schema.py

@@ -8,19 +8,19 @@
 
 from __future__ import annotations
 
-from typing import Optional
-
 from pydantic import BaseModel, Field
 
 
 class Decision(BaseModel):
     """The model's proposal for a single file."""
 
     filename: str = Field(description="Base name, NO extension; the tool re-adds the original")
-    subdir: str = Field(description="Target subfolder; one of the provided list unless is_new_subdir")
+    subdir: str = Field(
+        description="Target subfolder; one of the provided list unless is_new_subdir"
+    )
     is_new_subdir: bool = False
     doc_type: str = ""
-    date: Optional[str] = None  # ISO 'YYYY' / 'YYYY-MM' / 'YYYY-MM-DD'; null if unknown
+    date: str | None = None  # ISO 'YYYY' / 'YYYY-MM' / 'YYYY-MM-DD'; null if unknown
     summary: str = ""
     tags: list[str] = Field(default_factory=list)
     confidence: float = 0.0

scanfiler/apply.py

@@ -145,7 +145,8 @@ def undo(cfg: Config, *, run_id: str | None = None, last: bool = False) -> int:
     if not audit_file.is_file():
         return 0
 
-    records = [json.loads(line) for line in audit_file.read_text(encoding="utf-8").splitlines() if line.strip()]
+    lines = audit_file.read_text(encoding="utf-8").splitlines()
+    records = [json.loads(line) for line in lines if line.strip()]
     moves = [r for r in records if r.get("action") in ("copy", "move")]
     if not moves:
         return 0