From a2a977715095d4bca8c31ca495c45d8450b35a05 Mon Sep 17 00:00:00 2001 From: Lan Tian Date: Wed, 5 Aug 2026 17:44:45 +0800 Subject: [PATCH] feat: add V2 production e2e to the release gate Mirror tests/contract/test_v2_smoke.py against the live production V2 API (api.ade.landing.ai) as tests/contract/test_v2_production.py, marked `production`. Covers parse, extract, ground, and both job types (the soft-hidden build-schema is excluded, matching the staging smoke). The `production` marker means the existing e2e-production.yml run (`pytest -m production`) picks these up automatically, so V2 now gates release alongside V1. Rename the workflow "E2E Production (V1)" -> "E2E Production" and update the cost/label comments accordingly. Co-Authored-By: Claude Opus 5 --- .github/workflows/e2e-production.yml | 27 +++--- .github/workflows/release.yml | 2 +- tests/contract/test_v2_production.py | 127 +++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 13 deletions(-) create mode 100644 tests/contract/test_v2_production.py diff --git a/.github/workflows/e2e-production.yml b/.github/workflows/e2e-production.yml index 9b608f0..da5c63d 100644 --- a/.github/workflows/e2e-production.yml +++ b/.github/workflows/e2e-production.yml @@ -1,17 +1,19 @@ -# V1 end-to-end tests against the LIVE PRODUCTION API (api.va.landing.ai). +# V1 + V2 end-to-end tests against the LIVE PRODUCTION API. V1 targets api.va.landing.ai +# and V2 (`client.v2`) targets api.ade.landing.ai; the same key authenticates both hosts. # # Unlike the staging contract gate in pr-gates.yml, this is NOT wired to pull requests. # It runs in exactly two places: -# 1. manually — Actions -> "E2E Production (V1)" -> Run workflow; +# 1. manually — Actions -> "E2E Production" -> Run workflow; # 2. as the pre-tag gate in release.yml, which calls this workflow via `workflow_call` # and refuses to stamp/commit/tag if it goes red. # # COST: every run exercises the full V1 surface (parse, extract, build-schema, classify, -# section, split, and both job types) against production, so it spends real inference -# credits and leaves real job records in the production org — roughly 8 inference calls -# on a 2-page PDF. That is the deliberate price of gating releases on the real API; do -# not add this to a per-PR trigger. -name: E2E Production (V1) +# section, split, both job types) plus the V2 surface (parse, extract, ground, both job +# types — the soft-hidden build-schema is excluded), against production. It spends real +# inference credits and leaves real job records in the production org — roughly a dozen +# inference calls on a 2-page PDF. That is the deliberate price of gating releases on the +# real API; do not add this to a per-PR trigger. +name: E2E Production on: workflow_dispatch: @@ -25,7 +27,7 @@ concurrency: jobs: e2e-production: - name: e2e (production V1) + name: e2e (production V1 + V2) runs-on: ubuntu-latest timeout-minutes: 30 # Defense in depth: only main should reach the production key. release.yml runs on main @@ -75,11 +77,12 @@ jobs: - name: Install dependencies run: rye sync --all-features - - name: V1 e2e vs production + - name: V1 + V2 e2e vs production env: LANDINGAI_ADE_PRODUCTION_APIKEY: ${{ secrets.LANDINGAI_ADE_PRODUCTION_APIKEY }} # `-m production` overrides the `-m 'not contract and not production'` default in - # pyproject.toml. -n 0 forces serial execution: these hit the live API, and the - # repo's default `-n auto` (xdist) would run them in parallel, inviting flakiness - # and rate-limits — and would re-parse the sample document once per worker. + # pyproject.toml and selects both test_v1_production.py and test_v2_production.py. + # -n 0 forces serial execution: these hit the live API, and the repo's default + # `-n auto` (xdist) would run them in parallel, inviting flakiness and rate-limits + # — and would re-parse the sample document once per worker. run: rye run pytest tests/contract -m production -n 0 -v diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7ebfd74..222b8d4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ # # A maintainer runs this workflow (Actions -> Release -> Run workflow) and # picks the bump: patch / minor / major. The workflow then, in one shot: -# 0. runs the V1 production e2e suite (.github/workflows/e2e-production.yml) +# 0. runs the V1 + V2 production e2e suite (.github/workflows/e2e-production.yml) # against the live API — nothing below runs if it goes red, # 1. computes the next version from the latest git tag, # 2. stamps it into pyproject.toml and src/landingai_ade/_version.py, diff --git a/tests/contract/test_v2_production.py b/tests/contract/test_v2_production.py new file mode 100644 index 0000000..cc6047c --- /dev/null +++ b/tests/contract/test_v2_production.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import os +from typing import Iterator +from pathlib import Path + +import pytest +from pydantic import Field, BaseModel + +from landingai_ade import LandingAIADE +from landingai_ade.types.v2 import ( + JobStatus, + V2GroundResult, + V2ExtractResult, + V2ParseResponse, +) + +# `production` marker, same as the V1 production suite: excluded from the default `pytest` +# run and the staging contract gate, run only by e2e-production.yml (manual dispatch + the +# release gate). Mirrors tests/contract/test_v2_smoke.py against the LIVE PRODUCTION V2 API +# (api.ade.landing.ai) and spends real credits. The soft-hidden `v2.build_schema` surface is +# intentionally not covered here, matching the staging smoke. +pytestmark = pytest.mark.production + +PRODUCTION_KEY = os.environ.get("LANDINGAI_ADE_PRODUCTION_APIKEY") + +SAMPLE_PDF = Path(__file__).parent / "sample.pdf" + +# A tiny self-contained markdown document so extract can run without a parse first. +SAMPLE_MARKDOWN = "# Acme Inc. — Q1 Report\n\nTotal revenue for the quarter was **$1,250,000**.\n" + + +class RevenueSchema(BaseModel): + """Passing a pydantic model as the extract schema (V2 coerces it to JSON Schema).""" + + revenue: str = Field(description="The total revenue figure, verbatim") + company: str = Field(description="The company name") + + +@pytest.fixture(scope="module") +def production_client() -> Iterator[LandingAIADE]: + if not PRODUCTION_KEY: + pytest.skip("LANDINGAI_ADE_PRODUCTION_APIKEY not set") + # `environment="production"` routes V2 to api.ade.landing.ai (the V1 api.va host is unused + # by this suite). The same key authenticates both hosts. + with LandingAIADE(apikey=PRODUCTION_KEY, environment="production") as client: + yield client + + +@pytest.fixture(scope="module") +def parsed(production_client: LandingAIADE) -> V2ParseResponse: + """One real V2 parse, shared by the parse-shape and ground tests to hold spend down. + + The inline-grounding and parse-job tests issue their own calls (different options / + endpoint), so they do not use this. + """ + return production_client.v2.parse(document=SAMPLE_PDF) + + +def test_extract_sync(production_client: LandingAIADE) -> None: + res = production_client.v2.extract(schema=RevenueSchema, markdown=SAMPLE_MARKDOWN) + assert isinstance(res, V2ExtractResult) + assert isinstance(res.extraction, dict) + assert res.extraction + # `version` was renamed to `model_version` upstream; the current gateway populates it. + assert res.metadata.model_version + + +def test_extract_jobs(production_client: LandingAIADE) -> None: + job = production_client.v2.extract_jobs.create(schema=RevenueSchema, markdown=SAMPLE_MARKDOWN) + done = production_client.v2.extract_jobs.wait(job.job_id, timeout=300) + assert done.status is JobStatus.COMPLETED + assert isinstance(done.result, V2ExtractResult) + # Inline job: the metadata rides on `result.metadata`; the top-level `Job.metadata` + # receipt is only populated for `output_save_url` deliveries. + assert done.metadata is None + assert done.result.metadata.model_version + + +def test_parse_sync(parsed: V2ParseResponse) -> None: + assert isinstance(parsed, V2ParseResponse) + assert isinstance(parsed.markdown, str) + assert parsed.markdown + + +def test_parse_sync_inline_grounding_and_metadata(production_client: LandingAIADE) -> None: + # Exercise the current parse surface: `inline_markdown` option, per-node spatial + # `grounding` ({page, range, box}) inline on `structure`, and the renamed + # `output_markdown_chars` / `range_units` metadata fields. + resp = production_client.v2.parse(document=SAMPLE_PDF, options={"inline_markdown": True}) + assert isinstance(resp, V2ParseResponse) + assert resp.structure is not None and resp.structure.children + page = resp.structure.children[0] + assert page.grounding is not None and page.grounding.range is not None + assert page.grounding.box is not None + assert resp.metadata is not None + assert resp.metadata.range_units == "unicode_codepoints" + assert resp.metadata.output_markdown_chars is not None + + +def test_ground_sync(production_client: LandingAIADE, parsed: V2ParseResponse) -> None: + # Ground is a stateless join: extract against the parsed markdown, then ground the + # extraction back onto the parse structure the markdown came from. + assert parsed.structure is not None + extracted = production_client.v2.extract(schema=RevenueSchema, markdown=parsed.markdown or "") + grounded = production_client.v2.ground( + extraction_metadata=extracted.extraction_metadata, + structure=parsed.structure, + ) + assert isinstance(grounded, V2GroundResult) + assert isinstance(grounded.grounding, dict) + assert grounded.metadata.job_id + + +def test_parse_jobs(production_client: LandingAIADE) -> None: + job = production_client.v2.parse_jobs.create(document=SAMPLE_PDF) + done = production_client.v2.parse_jobs.wait(job.job_id, timeout=300) + assert done.status is JobStatus.COMPLETED + # Assert the normalized job result, not just the terminal status, so this actually + # covers the parse-job response contract (data -> V2ParseResponse). + assert isinstance(done.result, V2ParseResponse) + assert isinstance(done.result.markdown, str) + assert done.result.markdown + # Inline delivery: metadata rides on `result.metadata`, so the top-level `Job.metadata` + # receipt (set only for `output_save_url` deliveries) is absent. + assert done.metadata is None + assert done.result.metadata is not None