Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 15 additions & 12 deletions .github/workflows/e2e-production.yml
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
# V1 end-to-end tests against the LIVE PRODUCTION API (api.va.landing.ai).
# V1 + V2 end-to-end tests against the LIVE PRODUCTION API. V1 targets api.va.landing.ai
# and V2 (`client.v2`) targets api.ade.landing.ai; the same key authenticates both hosts.
#
# Unlike the staging contract gate in pr-gates.yml, this is NOT wired to pull requests.
# It runs in exactly two places:
# 1. manually — Actions -> "E2E Production (V1)" -> Run workflow;
# 1. manually — Actions -> "E2E Production" -> Run workflow;
# 2. as the pre-tag gate in release.yml, which calls this workflow via `workflow_call`
# and refuses to stamp/commit/tag if it goes red.
#
# COST: every run exercises the full V1 surface (parse, extract, build-schema, classify,
# section, split, and both job types) against production, so it spends real inference
# credits and leaves real job records in the production org — roughly 8 inference calls
# on a 2-page PDF. That is the deliberate price of gating releases on the real API; do
# not add this to a per-PR trigger.
name: E2E Production (V1)
# section, split, both job types) plus the V2 surface (parse, extract, ground, both job
# types — the soft-hidden build-schema is excluded), against production. It spends real
# inference credits and leaves real job records in the production org — roughly a dozen
# inference calls on a 2-page PDF. That is the deliberate price of gating releases on the
# real API; do not add this to a per-PR trigger.
name: E2E Production

on:
workflow_dispatch:
Expand All @@ -25,7 +27,7 @@ concurrency:

jobs:
e2e-production:
name: e2e (production V1)
name: e2e (production V1 + V2)
runs-on: ubuntu-latest
timeout-minutes: 30
# Defense in depth: only main should reach the production key. release.yml runs on main
Expand Down Expand Up @@ -75,11 +77,12 @@ jobs:
- name: Install dependencies
run: rye sync --all-features

- name: V1 e2e vs production
- name: V1 + V2 e2e vs production
env:
LANDINGAI_ADE_PRODUCTION_APIKEY: ${{ secrets.LANDINGAI_ADE_PRODUCTION_APIKEY }}
# `-m production` overrides the `-m 'not contract and not production'` default in
# pyproject.toml. -n 0 forces serial execution: these hit the live API, and the
# repo's default `-n auto` (xdist) would run them in parallel, inviting flakiness
# and rate-limits — and would re-parse the sample document once per worker.
# pyproject.toml and selects both test_v1_production.py and test_v2_production.py.
# -n 0 forces serial execution: these hit the live API, and the repo's default
# `-n auto` (xdist) would run them in parallel, inviting flakiness and rate-limits
# — and would re-parse the sample document once per worker.
run: rye run pytest tests/contract -m production -n 0 -v
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
#
# A maintainer runs this workflow (Actions -> Release -> Run workflow) and
# picks the bump: patch / minor / major. The workflow then, in one shot:
# 0. runs the V1 production e2e suite (.github/workflows/e2e-production.yml)
# 0. runs the V1 + V2 production e2e suite (.github/workflows/e2e-production.yml)
# against the live API — nothing below runs if it goes red,
# 1. computes the next version from the latest git tag,
# 2. stamps it into pyproject.toml and src/landingai_ade/_version.py,
Expand Down
127 changes: 127 additions & 0 deletions tests/contract/test_v2_production.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
from __future__ import annotations

import os
from typing import Iterator
from pathlib import Path

import pytest
from pydantic import Field, BaseModel

from landingai_ade import LandingAIADE
from landingai_ade.types.v2 import (
JobStatus,
V2GroundResult,
V2ExtractResult,
V2ParseResponse,
)

# `production` marker, same as the V1 production suite: excluded from the default `pytest`
# run and the staging contract gate, run only by e2e-production.yml (manual dispatch + the
# release gate). Mirrors tests/contract/test_v2_smoke.py against the LIVE PRODUCTION V2 API
# (api.ade.landing.ai) and spends real credits. The soft-hidden `v2.build_schema` surface is
# intentionally not covered here, matching the staging smoke.
pytestmark = pytest.mark.production

PRODUCTION_KEY = os.environ.get("LANDINGAI_ADE_PRODUCTION_APIKEY")

SAMPLE_PDF = Path(__file__).parent / "sample.pdf"

# A tiny self-contained markdown document so extract can run without a parse first.
SAMPLE_MARKDOWN = "# Acme Inc. — Q1 Report\n\nTotal revenue for the quarter was **$1,250,000**.\n"


class RevenueSchema(BaseModel):
"""Passing a pydantic model as the extract schema (V2 coerces it to JSON Schema)."""

revenue: str = Field(description="The total revenue figure, verbatim")
company: str = Field(description="The company name")


@pytest.fixture(scope="module")
def production_client() -> Iterator[LandingAIADE]:
if not PRODUCTION_KEY:
pytest.skip("LANDINGAI_ADE_PRODUCTION_APIKEY not set")
# `environment="production"` routes V2 to api.ade.landing.ai (the V1 api.va host is unused
# by this suite). The same key authenticates both hosts.
with LandingAIADE(apikey=PRODUCTION_KEY, environment="production") as client:
yield client


@pytest.fixture(scope="module")
def parsed(production_client: LandingAIADE) -> V2ParseResponse:
"""One real V2 parse, shared by the parse-shape and ground tests to hold spend down.

The inline-grounding and parse-job tests issue their own calls (different options /
endpoint), so they do not use this.
"""
return production_client.v2.parse(document=SAMPLE_PDF)


def test_extract_sync(production_client: LandingAIADE) -> None:
res = production_client.v2.extract(schema=RevenueSchema, markdown=SAMPLE_MARKDOWN)
assert isinstance(res, V2ExtractResult)
assert isinstance(res.extraction, dict)
assert res.extraction
# `version` was renamed to `model_version` upstream; the current gateway populates it.
assert res.metadata.model_version


def test_extract_jobs(production_client: LandingAIADE) -> None:
job = production_client.v2.extract_jobs.create(schema=RevenueSchema, markdown=SAMPLE_MARKDOWN)
done = production_client.v2.extract_jobs.wait(job.job_id, timeout=300)
assert done.status is JobStatus.COMPLETED
assert isinstance(done.result, V2ExtractResult)
# Inline job: the metadata rides on `result.metadata`; the top-level `Job.metadata`
# receipt is only populated for `output_save_url` deliveries.
assert done.metadata is None
assert done.result.metadata.model_version


def test_parse_sync(parsed: V2ParseResponse) -> None:
assert isinstance(parsed, V2ParseResponse)
assert isinstance(parsed.markdown, str)
assert parsed.markdown


def test_parse_sync_inline_grounding_and_metadata(production_client: LandingAIADE) -> None:
# Exercise the current parse surface: `inline_markdown` option, per-node spatial
# `grounding` ({page, range, box}) inline on `structure`, and the renamed
# `output_markdown_chars` / `range_units` metadata fields.
resp = production_client.v2.parse(document=SAMPLE_PDF, options={"inline_markdown": True})
assert isinstance(resp, V2ParseResponse)
assert resp.structure is not None and resp.structure.children
page = resp.structure.children[0]
assert page.grounding is not None and page.grounding.range is not None
assert page.grounding.box is not None
assert resp.metadata is not None
assert resp.metadata.range_units == "unicode_codepoints"
assert resp.metadata.output_markdown_chars is not None


def test_ground_sync(production_client: LandingAIADE, parsed: V2ParseResponse) -> None:
# Ground is a stateless join: extract against the parsed markdown, then ground the
# extraction back onto the parse structure the markdown came from.
assert parsed.structure is not None
extracted = production_client.v2.extract(schema=RevenueSchema, markdown=parsed.markdown or "")
grounded = production_client.v2.ground(
extraction_metadata=extracted.extraction_metadata,
structure=parsed.structure,
)
assert isinstance(grounded, V2GroundResult)
assert isinstance(grounded.grounding, dict)
assert grounded.metadata.job_id


def test_parse_jobs(production_client: LandingAIADE) -> None:
job = production_client.v2.parse_jobs.create(document=SAMPLE_PDF)
done = production_client.v2.parse_jobs.wait(job.job_id, timeout=300)
assert done.status is JobStatus.COMPLETED
# Assert the normalized job result, not just the terminal status, so this actually
# covers the parse-job response contract (data -> V2ParseResponse).
assert isinstance(done.result, V2ParseResponse)
assert isinstance(done.result.markdown, str)
assert done.result.markdown
# Inline delivery: metadata rides on `result.metadata`, so the top-level `Job.metadata`
# receipt (set only for `output_save_url` deliveries) is absent.
assert done.metadata is None
assert done.result.metadata is not None