diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..cf6cab5 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,71 @@ +# Live integration suite (issue #175): drives the real CLI against +# production — auth, parse, extract, then the local verbs (find, crop) +# over what production returned — on macOS and Windows, the two +# platforms customers install on. Three ways in: +# - every push to main (a merged PR that breaks production integration +# surfaces immediately, not at the next release), +# - the release pipeline calls it (workflow_call) and won't tag or +# build until it passes, +# - Actions -> Integration -> "Run workflow" for a manual check. +# Never on pull_request: every run bills real parse + extract credits +# (one of each per OS), and fork PRs must not reach the secret. +# The tests live in tests/integration/ and skip themselves unless +# ADE_INTEGRATION_API_KEY is set, so the offline suite stays hermetic. +# Contract-tested by tests/test_release_pipeline.py. +name: Integration + +on: + push: + branches: [main] + workflow_dispatch: + workflow_call: + secrets: + ADE_INTEGRATION_API_KEY: + description: Production ADE API key the suite authenticates with + required: true + +jobs: + integration: + # One matrix, two independent check rows ("Integration / macOS", + # "Integration / Windows"): fail-fast off so one platform's failure + # never cancels or masks the other's result. + name: ${{ matrix.platform }} + strategy: + fail-fast: false + matrix: + include: + - platform: macOS + runner: macos-latest + - platform: Windows + runner: windows-latest + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + steps: + # workflow_call enforces the secret via `required: true`, but a + # direct workflow_dispatch does not: a missing secret expands to "" + # and the whole suite would skip itself — a green run that never + # touched production. Fail loudly instead. + - name: Require the production API key + shell: bash + env: + ADE_INTEGRATION_API_KEY: ${{ secrets.ADE_INTEGRATION_API_KEY }} + run: | + if [ -z "$ADE_INTEGRATION_API_KEY" ]; then + echo "::error::ADE_INTEGRATION_API_KEY secret is empty or unset — the suite would silently skip every test" + exit 1 + fi + + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + python-version: "3.13" + + - name: Sync dependencies + run: uv sync --locked + + - name: Run integration suite against production + env: + ADE_INTEGRATION_API_KEY: ${{ secrets.ADE_INTEGRATION_API_KEY }} + run: uv run pytest tests/integration -v diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6638f20..143cef4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,6 +6,10 @@ # - Actions -> Release -> "Run workflow" on main (no clone needed): the # workflow tags v itself. The tagging must live in # here — a tag pushed with GITHUB_TOKEN never triggers workflows. +# Either way the live integration suite (integration.yml: auth, parse, +# extract, find, crop against production on macOS + Windows) must pass +# first — check depends on it, so a dispatch release isn't even tagged +# until production integration is green. # Contract-tested by tests/test_release_pipeline.py. name: Release @@ -18,7 +22,16 @@ permissions: contents: write jobs: + # The release gate (issue #175): the live macOS + Windows suite against + # production. First on purpose — under manual dispatch `check` pushes + # the release tag, and a failed gate must leave no tag behind. + integration: + uses: ./.github/workflows/integration.yml + secrets: + ADE_INTEGRATION_API_KEY: ${{ secrets.ADE_INTEGRATION_API_KEY }} + check: + needs: integration runs-on: ubuntu-latest outputs: tag: ${{ steps.version.outputs.tag }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b725b15..bdc6412 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,6 +15,22 @@ uv run pytest -q Tests are fully offline (fake transport). Lint config lives in `pyproject.toml` (`ruff`). +The one exception is `tests/integration/` — a live suite that drives +the CLI as real subprocesses against production (auth, parse, extract, +find, crop). It skips itself entirely unless `ADE_INTEGRATION_API_KEY` +holds a production API key, so it never runs by accident; when it does +run it bills real credits (one parse + one extract). Run it on demand: + +```sh +ADE_INTEGRATION_API_KEY= uv run pytest tests/integration -v +``` + +In CI it runs via `.github/workflows/integration.yml` on macOS and +Windows — on every push to `main` (a merged PR that breaks production +integration surfaces immediately), manually from **Actions → +Integration → "Run workflow"**, and as the release gate (below). Never +on pull requests: PR CI (`ci.yml`) runs the offline suite only. + ## Install from source (no clone) ```sh @@ -57,7 +73,10 @@ release is the tag `v`: tags `v` itself, refusing an already-released version), or locally: `git tag v0.3.0 && git push origin v0.3.0`. -The `Release` workflow (`.github/workflows/release.yml`) refuses a tag +The `Release` workflow (`.github/workflows/release.yml`) first runs the +live integration suite against production on macOS and Windows +(`integration.yml`, secret `ADE_INTEGRATION_API_KEY`) — a dispatch release +isn't even tagged until it passes. It then refuses a tag that doesn't match `pyproject.toml`, re-runs the test suite, builds a standalone PyInstaller app for each of the six platforms (macOS / Linux / Windows × arm64 / x86_64), runs `ade version` on every one, and diff --git a/tests/integration/fixtures/invoice.pdf b/tests/integration/fixtures/invoice.pdf new file mode 100644 index 0000000..fbcb913 Binary files /dev/null and b/tests/integration/fixtures/invoice.pdf differ diff --git a/tests/integration/test_production.py b/tests/integration/test_production.py new file mode 100644 index 0000000..9bc838d --- /dev/null +++ b/tests/integration/test_production.py @@ -0,0 +1,240 @@ +"""Live integration suite: the release gate against production. + +Unlike the offline suite (in-process runner, faked transport), every test +here drives the CLI as a real subprocess against the production ADE +service — auth verification, parse and extract submits/polls, then the +local verbs (find, crop) over what production returned. Run by +.github/workflows/integration.yml on macOS and Windows; the release +pipeline must pass it before anything is tagged or published (issue #175). + +Gated on ``ADE_INTEGRATION_API_KEY``: a production API key. Without it +every test skips, so plain ``pytest`` stays hermetic and free. To run +locally:: + + ADE_INTEGRATION_API_KEY= uv run pytest tests/integration -v + +The suite is one workflow in file order — login once, parse the fixture +once (session fixtures), then each verb asserts against that shared job +item. The fixture PDF's text is fixed (see FIXTURE_LINES), so content +assertions are exact, not model-lenient — except extraction values, +where only the trivially-quoted invoice number is asserted. + +Parse and extract bill real credits (one parse + one extract per run, +per OS). Everything else is served from the local store. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +API_KEY = os.environ.get("ADE_INTEGRATION_API_KEY", "") + +pytestmark = pytest.mark.skipif( + not API_KEY, reason="ADE_INTEGRATION_API_KEY not set (live production suite)" +) + +FIXTURE = Path(__file__).parent / "fixtures" / "invoice.pdf" +# The text baked into fixtures/invoice.pdf (regenerate: see the docstring +# there is none — the PDF is hand-assembled; keep these in sync with it). +FIXTURE_LINES = [ + "ADE CLI Integration Fixture", + "Invoice Number: INV-2026-0806", + "Customer: Example Corp", + "Total Due: 123.45 USD", +] +INVOICE_NUMBER = "INV-2026-0806" + +# Generous per-command ceiling: parse/extract poll server-side runs (the +# CLI's own --wait default is 600s). A hang, not slowness, is the failure +# this guards. +COMMAND_TIMEOUT = 900.0 + + +def run_ade( + args: list[str], + home: Path, + *, + stdin: str | None = None, +) -> subprocess.CompletedProcess[str]: + """Run the real CLI out of process, homed at a temp store, targeting + production. Every ADE_* variable is dropped so the run can't inherit a + developer's ADE_API_KEY/ADE_ENV/ADE_ENDPOINT — the stored credential + written by the login test is the only auth in play.""" + env = {k: v for k, v in os.environ.items() if not k.startswith("ADE_")} + env["ADE_HOME"] = str(home) + return subprocess.run( + [sys.executable, "-m", "ade_cli", *args], + capture_output=True, + text=True, + encoding="utf-8", + env=env, + input=stdin, + timeout=COMMAND_TIMEOUT, + ) + + +def payload_of(result: subprocess.CompletedProcess[str]) -> dict: + assert result.returncode == 0, ( + f"exit {result.returncode}\nstdout: {result.stdout}\nstderr: {result.stderr}" + ) + return json.loads(result.stdout) + + +@pytest.fixture(scope="session") +def home(tmp_path_factory: pytest.TempPathFactory) -> Path: + return tmp_path_factory.mktemp("ade-integration-home") + + +@pytest.fixture(scope="session") +def logged_in(home: Path) -> Path: + """Login once for the whole suite. `--api-key -` reads the key from + piped stdin — the headless path agents use, and the one that broke on + Windows in v1.0.2 — and verifies it live against production before + storing (ADR-0007).""" + result = run_ade( + ["login", "--api-key", "-", "--json"], home, stdin=API_KEY + "\n" + ) + payload = payload_of(result) + assert payload["verified"] is True # ADR-0007: verified live before storing + assert payload["stored"] is True + assert payload["environment"] == "production" + return home + + +@pytest.fixture(scope="session") +def parsed(logged_in: Path) -> dict: + """One real parse of the fixture; every downstream verb reads it.""" + return payload_of( + run_ade(["parse", "-d", str(FIXTURE), "--json"], logged_in) + ) + + +def test_auth_status_reports_the_stored_key(logged_in: Path) -> None: + payload = payload_of(run_ade(["auth", "status", "--json"], logged_in)) + assert payload["authenticated"] is True + assert payload["method"] == "api_key" + assert payload["source"] == "stored" + assert payload["environment"] == "production" + assert payload["credential"].endswith(API_KEY[-4:]) + + +def test_login_rejects_an_invalid_key(logged_in: Path, tmp_path: Path) -> None: + """Auth isn't just the happy path: production's 401 must come back as + the one canonical invalid-key error, and nothing may be stored. A + fresh home so the real credential is never at risk.""" + bad_home = tmp_path / "bad-home" + result = run_ade( + ["login", "--api-key", "-", "--json"], bad_home, stdin="sk-not-a-real-key\n" + ) + assert result.returncode != 0 + payload = json.loads(result.stdout) + assert payload["error"] == "invalid_api_key" + assert payload["status_code"] == 401 + assert not (bad_home / "credentials.json").exists() + + +def test_parse_completes_against_production(parsed: dict) -> None: + assert parsed["status"] == "parsed" + assert parsed["environment"] == "production" + assert parsed["page_count"] == 1 + assert parsed["failed_pages"] == [] + assert parsed["job_item_id"] + assert parsed["run_id"] + markdown = Path(parsed["store_dir"], "parse.md").read_text(encoding="utf-8") + assert INVOICE_NUMBER in markdown + + +def test_parse_rerun_is_served_from_the_store(parsed: dict, logged_in: Path) -> None: + """The guarantee contract: the same invocation dedups to the same job + item and bills nothing the second time.""" + again = payload_of(run_ade(["parse", "-d", str(FIXTURE), "--json"], logged_in)) + assert again["job_item_id"] == parsed["job_item_id"] + assert again["cached"] is True + # The same server-side run — nothing was resubmitted, so nothing new + # billed. (`credits` echoes what the original run billed, so it is no + # signal here.) + assert again["run_id"] == parsed["run_id"] + + +def test_find_searches_the_parsed_elements(parsed: dict, logged_in: Path) -> None: + matches = payload_of( + run_ade( + ["find", parsed["job_item_id"], "invoice number", "--json"], logged_in + ) + ) + assert isinstance(matches, list) and matches + hit = matches[0] + assert hit["job_item_id"] == parsed["job_item_id"] + assert hit["page"] == 1 + assert INVOICE_NUMBER in hit["text"] + box = hit["box"] + assert 0 <= box["xmin"] < box["xmax"] <= 1 + assert 0 <= box["ymin"] < box["ymax"] <= 1 + + +def test_crop_renders_every_element_to_png( + parsed: dict, logged_in: Path, tmp_path: Path +) -> None: + out = tmp_path / "crops" + payload = payload_of( + run_ade( + [ + "crop", + parsed["job_item_id"], + "--all", + "--output", + str(out), + "--json", + ], + logged_in, + ) + ) + assert payload["status"] == "cropped" + assert payload["count"] == len(payload["crops"]) > 0 + for crop in payload["crops"]: + png = Path(crop["path"]) + assert png.is_file() + assert png.read_bytes()[:8] == b"\x89PNG\r\n\x1a\n" + assert crop["width"] > 0 and crop["height"] > 0 + + +def test_extract_pulls_the_invoice_number(parsed: dict, logged_in: Path) -> None: + schema = json.dumps( + { + "type": "object", + "properties": { + "invoice_number": { + "type": "string", + "description": "The invoice number, verbatim.", + } + }, + "required": ["invoice_number"], + } + ) + payload = payload_of( + run_ade( + ["extract", parsed["job_item_id"], "--schema", schema, "--json"], + logged_in, + ) + ) + assert payload["status"] == "extracted" + assert payload["environment"] == "production" + assert payload["parse_job_item_id"] == parsed["job_item_id"] + assert payload["job_item_id"] != parsed["job_item_id"] + assert payload["extraction"]["invoice_number"] == INVOICE_NUMBER + + +def test_logout_clears_the_credential(logged_in: Path) -> None: + """Last in file order — every earlier test rides the stored login.""" + payload = payload_of(run_ade(["logout", "--json"], logged_in)) + assert payload["cleared"] is True + assert payload["environment"] == "production" + status = run_ade(["auth", "status", "--json"], logged_in) + assert status.returncode == 1 # unauthenticated is a distinct exit state + assert json.loads(status.stdout)["authenticated"] is False diff --git a/tests/test_release_pipeline.py b/tests/test_release_pipeline.py index 448cc01..616519f 100644 --- a/tests/test_release_pipeline.py +++ b/tests/test_release_pipeline.py @@ -17,6 +17,7 @@ ROOT = Path(__file__).resolve().parents[1] WORKFLOW = ROOT / ".github" / "workflows" / "release.yml" +INTEGRATION = ROOT / ".github" / "workflows" / "integration.yml" INSTALL_SH = ROOT / "scripts" / "install.sh" INSTALL_PS1 = ROOT / "scripts" / "install.ps1" INSTALL_CMD = ROOT / "scripts" / "install.cmd" @@ -51,6 +52,50 @@ def test_release_workflow_is_manually_dispatchable(): assert "git tag" in text +def test_release_gates_on_the_live_integration_suite(): + """Issue #175: nothing ships without the production integration suite + passing. The gate must sit ahead of `check` — under manual dispatch + `check` pushes the release tag, so a failed gate must leave no tag — + and everything downstream inherits it through `needs`.""" + text = WORKFLOW.read_text(encoding="utf-8") + assert "uses: ./.github/workflows/integration.yml" in text + check_job = text.split("\n check:", 1)[1] + assert re.search(r"^\s*needs: integration\b", check_job, re.MULTILINE) + + +def test_integration_suite_runs_on_macos_and_windows(): + """The two platforms customers install on (issue #175). Post-merge + pushes to main run it (early signal) and manual dispatch stays + available, but pull_request must never trigger it — every run bills + real parse + extract credits against production, and fork PRs must + never reach the secret.""" + text = INTEGRATION.read_text(encoding="utf-8") + assert "workflow_dispatch" in text + assert "workflow_call" in text + assert re.search(r"push:\s*\n\s*branches: \[main\]", text) + # A trigger key, not prose: pull_request and pull_request_target both. + assert not re.search(r"^\s*pull_request\w*:", text, re.MULTILINE) + assert re.search(r"runner:\s*macos-", text) + assert re.search(r"runner:\s*windows-", text) + assert "ADE_INTEGRATION_API_KEY" in text + assert "tests/integration" in text + # An unset secret expands to "" under workflow_dispatch, the suite + # skips itself, and the run reads green having touched nothing — the + # workflow must fail loudly on an empty key instead. + assert 'if [ -z "$ADE_INTEGRATION_API_KEY" ]' in text + + +def test_integration_tests_skip_without_the_live_key(): + """The offline suite must stay hermetic: every test in + tests/integration/ hangs off the ADE_INTEGRATION_API_KEY gate.""" + for path in sorted((ROOT / "tests" / "integration").glob("test_*.py")): + text = path.read_text(encoding="utf-8") + assert "ADE_INTEGRATION_API_KEY" in text, f"{path.name} misses the gate" + assert re.search(r"^pytestmark = pytest\.mark\.skipif", text, re.MULTILINE), ( + f"{path.name} does not skip module-wide without the key" + ) + + def test_release_builds_onedir_not_onefile(): """--onefile re-extracts ~100 shared libraries to fresh inodes on every launch, so macOS re-validates every code signature every run — ~10s per