diff --git a/.github/workflows/check-python-code.yaml b/.github/workflows/check-python-code.yaml index d0248742c..c00001087 100644 --- a/.github/workflows/check-python-code.yaml +++ b/.github/workflows/check-python-code.yaml @@ -6,12 +6,16 @@ on: - 'packages/**' - 'pyproject.toml' - 'uv.lock' + - 'Makefile' + - '.github/workflows/check-python-code.yaml' push: branches: [main, dev] paths: - 'packages/**' - 'pyproject.toml' - 'uv.lock' + - 'Makefile' + - '.github/workflows/check-python-code.yaml' permissions: contents: read @@ -31,7 +35,8 @@ jobs: # Default resolution exercises the committed lock against every # supported Python minor version. The lowest-direct cell pins each # direct dependency to its declared floor (see UV_RESOLUTION below) - # and runs only on the Python floor. + # and runs only on the Python floor, since the resolved-low pyspark + # 3.4 wheels exist for 3.10/3.11 only. python: ["3.10", "3.11", "3.12", "3.13", "3.14"] resolution: [default] include: @@ -56,9 +61,19 @@ jobs: with: python-version: ${{ matrix.python }} + # PySpark 3.4 (the declared minimum) does not support Java 21, which is + # the default JDK on ubuntu-latest runners. Pin to Java 17 for the + # lowest-direct cell so the resolved pyspark==3.4.0 can actually start. + - name: Set up JDK 17 + if: matrix.resolution == 'lowest-direct' + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + distribution: temurin + java-version: '17' + # UV_RESOLUTION=lowest-direct makes `uv sync` re-resolve every direct # dependency to the lowest version permitted by pyproject.toml. This - # exercises the declared floor (e.g. pydantic==2.12.0) instead of + # exercises the declared floor (e.g. pyspark==3.4.0) instead of # whatever the committed lock happens to point at. Failures here mean # a direct dep's minimum needs to be bumped. Set via GITHUB_ENV only # in the relevant cell so default cells run with no UV_RESOLUTION at @@ -69,3 +84,14 @@ jobs: - name: Run make check run: make check + + # `make check` regenerates the committed PySpark output in place. If a PR + # changed the schema or codegen without committing the regenerated files, + # the working tree now differs from what was committed -- fail loudly so + # stale generated output cannot pass CI. + - name: Verify generated PySpark output is committed + run: | + git diff --exit-code -- \ + packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated \ + packages/overture-schema-pyspark/tests/generated \ + || { echo "::error::Generated PySpark output is stale. Run 'make generate-pyspark' and commit."; exit 1; } diff --git a/.gitignore b/.gitignore index 6a40dfafd..26c73e5f7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ docs/docusaurus __pycache__/ .coverage +.testmondata* diff --git a/Makefile b/Makefile index edea9b34d..d73734fd1 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,49 @@ -.PHONY: default uv-sync check test-all test test-only docformat doctest doctest-only mypy mypy-only lint-only update-baselines +.PHONY: default uv-sync clean-pyspark generate-pyspark verify-pyspark-generated check test-all test test-only docformat doctest doctest-only mypy mypy-only lint-only update-baselines + +TESTMON ?= --testmon default: test-all install: uv-sync uv-sync: - @uv sync --all-packages 2> /dev/null + @uv sync --all-packages --all-extras -q + +PYSPARK_EXPRESSIONS := packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated +PYSPARK_GENERATED_TESTS := packages/overture-schema-pyspark/tests/generated + +clean-pyspark: + @rm -rf $(PYSPARK_EXPRESSIONS) $(PYSPARK_GENERATED_TESTS) -check: uv-sync +generate-pyspark: uv-sync clean-pyspark + @uv run overture-codegen generate --format pyspark \ + --output-dir $(PYSPARK_EXPRESSIONS) \ + --test-output-dir $(PYSPARK_GENERATED_TESTS) + @uv run ruff check --fix --quiet $(PYSPARK_EXPRESSIONS) $(PYSPARK_GENERATED_TESTS) + @uv run ruff format --quiet $(PYSPARK_EXPRESSIONS) $(PYSPARK_GENERATED_TESTS) + +check: uv-sync generate-pyspark @$(MAKE) -j test-only doctest-only lint-only mypy-only +# Regenerate and fail if the committed generated output differs. Catches PRs +# that change the schema or codegen without committing the regenerated files -- +# `check` itself regenerates, so without this guard stale committed output is +# silently overwritten before the tests run and never verified. +verify-pyspark-generated: generate-pyspark + @git diff --exit-code -- $(PYSPARK_EXPRESSIONS) $(PYSPARK_GENERATED_TESTS) \ + || { echo "Generated PySpark output is stale; run 'make generate-pyspark' and commit."; exit 1; } + +# test-all is the unconditional full run -- testmon-independent, unlike the +# incremental test/test-only targets -- so data-only changes (golden JSON, +# [[examples]]) that testmon cannot see still get exercised. test-all: uv-sync @uv run pytest -W error packages/ test: uv-sync - @uv run pytest -W error packages/ -x -q --tb=short + @uv run pytest -W error $(TESTMON) packages/ -x -q --tb=short test-only: - @uv run pytest -W error packages/ -x -q --tb=short + @uv run pytest -W error $(TESTMON) packages/ -x -q --tb=short coverage: uv-sync @uv run pytest packages/ --cov overture.schema --cov-report=term --cov-report=html && open htmlcov/index.html diff --git a/packages/overture-schema-codegen/README.md b/packages/overture-schema-codegen/README.md index d83cb7967..61c029505 100644 --- a/packages/overture-schema-codegen/README.md +++ b/packages/overture-schema-codegen/README.md @@ -41,14 +41,14 @@ Rendering Output formatting, all presentation decisions ^ Output Layout What to generate, where it goes, how outputs link ^ -Extraction TypeInfo, FieldSpec, ModelSpec, UnionSpec +Extraction TypeInfo, FieldSpec, RecordSpec, UnionSpec ^ Discovery discover_models() from overture-schema-common ``` **Discovery** loads registered Pydantic models via entry points. The return dict includes both concrete `BaseModel` subclasses (like `Building`) and discriminated union -type aliases (like `Segment`). Both satisfy the `FeatureSpec` protocol and flow through +type aliases (like `Segment`). Both satisfy the `ModelSpec` protocol and flow through the same pipeline. **Extraction** unwraps type annotations into specs. `analyze_type()` is the central diff --git a/packages/overture-schema-codegen/docs/design.md b/packages/overture-schema-codegen/docs/design.md index 662d77fc5..ec4e3ec1a 100644 --- a/packages/overture-schema-codegen/docs/design.md +++ b/packages/overture-schema-codegen/docs/design.md @@ -30,9 +30,9 @@ definitions regularly nest `Annotated` inside `NewType` inside `Annotated` -- Annotated[int, Field(ge=...)])` -- and constraints at each depth need to be tagged with the NewType that contributed them. -The code generator solves this by extracting type information once into a flat, -navigable representation (`TypeInfo`), then passing that to renderers that produce -output without touching Python's type system. +The code generator solves this by extracting type information once into a tree-shaped +`FieldShape` IR, then passing that to renderers that produce output without touching +Python's type system. ## Inputs and Outputs @@ -41,10 +41,12 @@ points, plus example data from theme `pyproject.toml` files. Examples serve two purposes: rendered examples in documentation pages, and a starting point for generating tests that verify behavior of generated code. -**Current Outputs**: Markdown documentation pages with field tables, cross-page links, -constraint descriptions, and examples. +**Outputs**: -**Planned outputs**: Arrow schemas, PySpark expressions. +- Markdown documentation pages with field tables, cross-page links, constraint + descriptions, and examples. +- PySpark validation modules: per-model expression builders, StructType schemas, + a feature registry, and generated conformance test modules. ## Architecture @@ -55,15 +57,21 @@ Rendering Output formatting, all presentation decisions ^ Output Layout What to generate, where it goes, how outputs link ^ -Extraction TypeInfo, FieldSpec, ModelSpec, EnumSpec, ... +Extraction FieldShape, FieldSpec, RecordSpec, EnumSpec, ... ^ Discovery discover_models() from overture-schema-common ``` -`markdown/pipeline.py` orchestrates the pipeline without I/O: it expands feature trees, -collects supplementary types, builds placement registries, computes reverse references, -and calls renderers -- returning `RenderedPage` objects. The CLI (`cli.py`) is a thin -Click wrapper that calls `generate_markdown_pages()` and writes files to disk. +Each output format has its own pipeline module that orchestrates without I/O: + +- `markdown/pipeline.py` expands feature trees, collects supplementary types, builds + placement registries, computes reverse references, and calls renderers -- returning + `RenderedPage` objects. +- `pyspark/pipeline.py` expands feature trees, builds checks and schemas, renders + expression modules and test modules -- returning `GeneratedModule` objects. + +The CLI (`cli.py`) is a thin Click wrapper that dispatches to the appropriate pipeline +and writes files to disk. ```mermaid graph TD @@ -75,24 +83,24 @@ graph TD subgraph Extraction EX["extraction/type_analyzer / extractors"] - EX -->|"ModelSpec, UnionSpec"| TREE["expand_model_tree()"] end - TREE -->|"FeatureSpec[]"| OL + EX -->|"ModelSpec[]"| OL + EX -->|"ModelSpec[]"| PS - subgraph "Output Layout" + subgraph "Output Layout (Markdown)" OL["layout/type_collection"] OL -->|"SupplementarySpec{}"| PA["markdown/path_assignment"] PA -->|"dict[str, Path]"| LC["markdown/link_computation"] RR["markdown/reverse_references"] end - subgraph Rendering + subgraph "Markdown Rendering" R["markdown/renderer"] TR["extraction/type_registry"] -.->|"type name resolution"| R end - subgraph Orchestration + subgraph "Markdown Orchestration" MP["markdown/pipeline"] end @@ -101,56 +109,83 @@ graph TD RR --> MP MP --> R R -->|"RenderedPage[]"| MP + + subgraph "PySpark Pipeline" + PS["pyspark/pipeline"] + CD["constraint_dispatch"] -->|"ExpressionDescriptor"| CB + CB["check_builder"] -->|"Check, ModelCheck"| PR + SB["schema_builder"] -->|"SchemaField[]"| PR + CB -->|"Check, ModelCheck"| PTR + SY["test_data/"] -->|"BASE_ROW, scaffold, invalid_value"| PTR + PR["renderer"] + PTR["test_renderer"] + end + + PS --> CD + PS --> CB + PS --> SB + PS --> PR + PS --> PTR + MP -->|"list[RenderedPage]"| CLI["cli.py → disk"] + PS -->|"list[GeneratedModule]"| CLI ``` ## Extraction -### `analyze_type` -- iterative type unwrapping - -`analyze_type(annotation)` is a single iterative function that peels type annotation -layers in a fixed order, accumulating information into an `_UnwrapState`: - -1. **NewType**: Records the outermost name (user-facing semantic identity, e.g. - `FeatureVersion`) and updates the "current" name (used for constraint provenance and - as `base_type` at terminal) -2. **Annotated**: Collects constraints from metadata, each tagged with whichever NewType - was most recently entered. Extracts `Field.description` when present -3. **Union**: Filters out `None` (marks optional), `Sentinel`, and `Literal` sentinel - arms. If multiple concrete `BaseModel` arms remain, classifies as `UNION`; otherwise - continues with the single remaining arm -4. **list / dict**: Increments `list_depth` for each `list[...]` layer, sets dict flags, - continues into element types -5. **Terminal**: Classifies as `PRIMITIVE`, `LITERAL`, `ENUM`, `MODEL`, or `UNION` - -The result is `TypeInfo` -- a flat dataclass that fully describes the unwrapped type: -classification (`TypeKind`), optional/dict flags, `list_depth` (count of `list[...]` -layers), `newtype_outer_list_depth` (list layers outside the outermost NewType boundary), -accumulated constraints with provenance, NewType names, source type, literal values, and -(for UNION kind) the tuple of concrete `BaseModel` member types. Dict types carry -recursively analyzed `TypeInfo` for their key and value types. - -Multi-depth `Annotated` layers (common in practice, since NewTypes wrap `Annotated` -types that wrap further NewTypes) are handled naturally by the loop -- each iteration -processes the next wrapper. Constraints from each `Annotated` layer are tagged with the -NewType active at that depth. +### `analyze_type` -- recursive type unwrapping + +`analyze_type(annotation)` recurses through a Python type annotation, peeling one layer +per call frame via the internal `_unwrap` function: + +1. **NewType**: Constructs `_NewTypeCtx` with the NewType's name, recurses into + `__supertype__`, then wraps the result in `NewTypeShape`. `_erase_inner_newtypes` + strips every inner `NewTypeShape` reached through `ArrayOf` layers, so each spine + keeps only its outermost `NewTypeShape` (inner NewType names survive on the + terminal `Primitive.base_type`). +2. **Annotated**: Collects constraints from metadata as `ConstraintSource` objects, + each tagged with the active `_NewTypeCtx`. Extracts `Field.description` when present. + Recurses into the inner annotation, then attaches constraints to the result via + `attach_constraints`, which prepends them to the outermost structural layer. +3. **Union**: Delegates to `_peel_union`, which filters `None` (marks optional), + `Sentinel`, and `Literal` sentinel arms. Multiple concrete `BaseModel` arms invoke + `union_resolver`; a single arm continues with `_unwrap`. +4. **list / dict**: `list[X]` recurses into `X` and wraps in `ArrayOf`. Nested lists + produce nested `ArrayOf` instances -- no numeric depth counter. `dict[K, V]` recurses + for key and value independently and returns `MapOf`. +5. **Terminal**: Classifies as `Primitive`, `LiteralScalar`, `AnyScalar`, `ModelRef`, + or `UnionRef`. + +The result is `tuple[FieldShape, bool, str | None]` -- the structural shape describing +the type as a nested tree, whether the field accepts `None`, and the first +`FieldInfo.description` found during unwrapping. `FieldShape` is a discriminated union +of eight variants (`Primitive`, `LiteralScalar`, `AnyScalar`, `ModelRef`, `UnionRef`, +`ArrayOf`, `MapOf`, `NewTypeShape`) nested to describe arbitrary collection and NewType +wrapping. + +Constraints from each `Annotated` layer attach to the shape layer they annotate -- +`attach_constraints` walks past any `NewTypeShape` wrappers to prepend constraints on +the first `ArrayOf`, `MapOf`, or scalar node. This means array-level and element-level +constraints land on structurally distinct nodes without any numeric bookkeeping. ### Extractors by domain Extraction is split by entity kind: -- `extraction/model_extraction.py`: Pydantic model -> `ModelSpec` (fields in MRO-aware +- `extraction/model_extraction.py`: Pydantic model -> `RecordSpec` (fields in MRO-aware documentation order, alias-resolved names, model-level constraints) - `extraction/enum_extraction.py`: Enum class -> `EnumSpec` - `extraction/newtype_extraction.py`: NewType -> `NewTypeSpec` - `extraction/union_extraction.py`: Discriminated union alias -> `UnionSpec` - `extraction/numeric_extraction.py`: Numeric types -> `NumericSpec` +- `extraction/pydantic_extraction.py`: Pydantic built-in type -> `PydanticTypeSpec` -Each calls `analyze_type()` for field types. Tree expansion (`expand_model_tree()`) -walks MODEL-kind fields to populate nested model references, with a shared cache and -cycle detection (`starts_cycle=True`). +Each calls `analyze_type()` for field types. `extract_model` recurses into sub-models +and sub-unions during extraction, building `ModelRef`/`UnionRef` terminals with their +specs resolved. A shared cache and cycle detection (`starts_cycle=True`) prevent +infinite recursion and duplicate extraction. -### Unions and the FeatureSpec protocol +### Unions and ModelSpec Discriminated unions (e.g. `Segment = Annotated[Union[RoadSegment, ...], Discriminator(...)]`) are type aliases, not classes. `UnionSpec` captures the union @@ -159,10 +194,10 @@ Fields shared across all variants appear once; fields present in some variants a wrapped in `AnnotatedField` with `variant_sources` indicating which members contribute them. The common base class is identified so shared fields can be deduplicated. -`FeatureSpec` is a `Protocol` satisfied by both `ModelSpec` and `UnionSpec`. Code that -operates on "any top-level feature" -- tree expansion, supplementary type collection, -rendering dispatch -- uses `FeatureSpec` rather than a concrete type, so union and model -features flow through the same pipeline. +`ModelSpec` is a type alias `RecordSpec | UnionSpec`. Code that operates on "any +top-level model" -- supplementary type collection, rendering dispatch -- uses +`ModelSpec` so records and unions flow through the same pipeline. Consumers +narrow with `isinstance` when arm-specific attributes are needed. ### Constraints @@ -180,9 +215,12 @@ reference each other. ### Supplementary type collection -`collect_all_supplementary_types()` walks the expanded field trees of all feature specs, -extracting enums, semantic NewTypes, and sub-models that need their own output. Returns -`dict[str, SupplementarySpec]`. +`collect_all_supplementary_types()` walks the field trees of all feature specs to extract +the supplementary types that need their own output: enums, semantic NewTypes, sub-models, +and Pydantic built-in types (`HttpUrl`, `EmailStr`). Returns `dict[TypeIdentity, +SupplementarySpec]`, where `SupplementarySpec = EnumSpec | NewTypeSpec | RecordSpec | +PydanticTypeSpec`. `TypeIdentity` pairs a unique Python object with its display name so +registry lookups remain stable when two distinct types share a name. ### Module-mirrored output paths @@ -195,14 +233,14 @@ directory. ### Link computation -`LinkContext` carries the current output's path and the full type-to-path registry. When -a renderer formats a type reference, it looks up the target in the registry and computes -a relative path. Links exist only for types with registry entries, avoiding broken -references to ungenerated outputs. +`LinkContext` carries the current output's path and the full `dict[TypeIdentity, +PurePosixPath]` registry. When a renderer formats a type reference, it looks up the +target by `TypeIdentity` and computes a relative path. Links exist only for types with +registry entries, avoiding broken references to ungenerated outputs. ### Reverse references -`compute_reverse_references()` walks feature specs to build `dict[type_name, +`compute_reverse_references()` walks feature specs to build `dict[TypeIdentity, list[UsedByEntry]]` for "Used By" sections. ## Rendering @@ -221,10 +259,10 @@ to registered primitives. ### Markdown renderer Jinja2 templates for feature, enum, NewType, primitives, and geometry pages. -`render_feature()` expands MODEL-kind fields inline with dot-notation (e.g., -`sources[].dataset`), stopping at cycle boundaries. `format_type()` in -`markdown/type_format.py` converts `TypeInfo` into link-aware display strings using -`LinkContext`. +`render_model()` walks each field's `FieldShape` tree and expands `ModelRef` +terminals inline with dot-notation (e.g., `sources[].dataset`), stopping at +`ModelRef.starts_cycle`. `format_type()` in `markdown/type_format.py` converts a +`FieldShape` into link-aware display strings using `LinkContext`. ### Constraint prose @@ -247,17 +285,157 @@ need for external schema information -- the model instance itself encodes the ty structure. `augment_missing_fields` appends `(name, None)` entries for union cross-arm fields absent from the concrete variant instance. +## PySpark Pipeline + +The PySpark codegen transforms extracted `ModelSpec` trees into validation expression +modules and generated conformance test modules. `pyspark/pipeline.py` exposes +`generate_pyspark_module` (single spec) and `generate_pyspark_modules` (all specs). + +### Constraint Dispatch + +`pyspark/constraint_dispatch.py` maps constraint objects to expression descriptors. +Four dispatch mechanisms: + +1. **`dispatch_constraint`** -- field constraints (bounds, min/max length, pattern, + stripped, geometry type, unique items, JSON pointer). Returns `ExpressionDescriptor` + with function name, args, kwargs. Returns None for skipped constraints (Reference, + Strict). + +2. **`dispatch_newtype`** -- NewType-level overrides: `LinearlyReferencedRange` -> + three range checks. `CountryCodeAlpha2` and `RegionCode` decompose normally + via their `PatternConstraint` subclasses and return None here. + +3. **`dispatch_base_type`** -- base-type overrides for types with no `Annotated` + constraints: `HttpUrl` -> `check_url_format` + `check_url_length`, + `EmailStr` -> `check_email`, `BBox` -> `check_bbox_completeness` + + `check_bbox_lat_ordering` + `check_bbox_lat_range`. + +4. **`dispatch_model_constraint`** -- model constraints: `RequireAnyOfConstraint`, + `RadioGroupConstraint`, `RequireIfConstraint`, `ForbidIfConstraint`, + `MinFieldsSetConstraint`. Returns `ModelConstraintDescriptor`. Returns None for + `NoExtraFieldsConstraint`. + +### Check Builder + +`pyspark/check_builder.py` walks `FieldSpec` trees to produce `Check` and `ModelCheck` +IR. Resolves the mapping from nested field paths to PySpark array iteration patterns, +producing a `FieldPath` (`ScalarPath` or `ArrayPath`) on each `Check`: + +- **Scalar field** -- `ScalarPath`; renders as `F.col("field")` +- **Top-level array** -- `ArrayPath` with one `ArraySegment`; renders as + `array_check("field", lambda el: ...)` +- **Field inside an array element** -- `ArrayPath` with struct navigation after the + array segment; renders as `array_check("array_col", lambda el: el["field"])` +- **Nested array inside an array** -- `ArrayPath` with multiple `ArraySegment`s; + renders as `nested_array_check("outer", lambda el: array_check(el["inner"], ...))` +- **Multiple nesting levels** -- chained `nested_array_check` with struct segments + navigating between array iterations + +Union handling: variant-specific fields are annotated with `ColumnGuard` or +`ElementGuard` discriminator gates. `Check.guards` is AND-composed at render time. +Nested unions (a union field within a union) produce a `ColumnGuard` and an +`ElementGuard` in sequence on the same check. + +`COLUMN_LEVEL_FUNCTIONS` (frozenset) selects checks that split into a +separate `Check`; `_COLUMN_LEVEL_SUFFIXES` (dict) supplies the label +suffix for each: `check_required` (no suffix), `check_array_min_length` +(`_min_length`), `check_array_max_length` (`_max_length`), +`check_struct_unique` (`_unique`). + +### Schema Builder + +`pyspark/schema_builder.py` converts `FieldSpec` trees to `SchemaField` lists for +StructType source generation. Maps types to Spark type expressions via the type registry. +`SHARED_TYPE_REFS` reserves a few base-type names for `_schema_structs.py` constants +when the codegen cannot walk the type -- currently just `BBox` -> `BBOX_STRUCT` (BBox +is a plain class, not a Pydantic `BaseModel`). Pydantic models are inlined into the +StructType expression. Union fields are deduplicated by name with type widening (the +wider Spark numeric type wins). + +### Renderer + +`pyspark/renderer.py` emits per-model Python modules containing: + +- Private `_fieldname_check()` functions returning `Check(field=, name=, expr=, shape=, root_field=)` +- A public `_checks() -> list[Check]` function calling all of them +- A per-model `MODELNAME_SCHEMA` StructType constant (e.g. `ADDRESS_SCHEMA`, `SEGMENT_SCHEMA`) +- An `ENTRY_POINT` string, a `PARTITIONS` dict describing the feature's Hive partition + layout (empty when not partitioned), and a `MODEL_VALIDATION` constant pairing the + schema and checks + +The registry is not generated. `_registry.py` lives hand-written in the +`overture-schema-pyspark` package and walks the `expressions.generated` namespace at +import time, collecting every module that exposes `ENTRY_POINT` and `MODEL_VALIDATION` +into a `dict[str, ModelValidation]`. Modules that also expose `PARTITIONS` populate a +parallel partition map keyed by entry point. + +Expression rendering handles scalar expressions, array_check/nested_array_check chains, +variant gating (`F.when(discriminator.isin(...))`), nullable parent gating +(`F.when(gate.isNotNull(), ...)`), and nested lambda variable naming for deep nesting. +Output is formatted with ruff. + +### Test Renderer + +`pyspark/test_renderer.py` emits per-model pytest modules containing: + +- `BASE_ROW_SPARSE` / `BASE_ROW_POPULATED` -- valid synthetic rows +- `SCENARIOS: list[Scenario]` -- generated test cases, each carrying a + `mutate` callable that produces an invalid row from a merged base +- Fixtures: `checks`, `sparse_results`, `populated_results` +- Tests: `test_baseline_sparse`, `test_baseline_populated`, + `test_scenario_sparse`, `test_scenario_populated` (parametrized). + Schema coverage runs inside `run_validation_pipeline` via + `assert_schema_covers_checks`, not in a separate test. + +Union specs with multiple discriminator arms produce one test module per arm. + +### Test Data Generator + +`pyspark/test_data/` is a subpackage with three modules: + +- `base_row.py` -- `generate_base_row` / `generate_populated_row` produce sparse + (required only) and fully populated valid rows from a `ModelSpec`. Consults field + constraints to produce constraint-satisfying values (country codes, geometry WKT, + bounds-respecting numbers). `generate_arm_rows` / `generate_populated_arm_rows` + produce one row per discriminator arm for union specs. +- `scaffold.py` -- `generate_scaffold` / `generate_model_scaffold` build sparse dicts + that provide nested structure (optional structs, arrays) needed for test scenarios. +- `invalid_value.py` -- `invalid_value` produces a concrete value that violates each + check function. + +### Known Semantic Gaps + +PySpark validation diverges from Pydantic validation in two documented areas: + +- `UniqueItemsConstraint` uses Spark's `array_distinct`, which compares whole + elements with structural equality (struct- and nested-array-aware) on the raw + stored values. Pydantic compares normalized Python objects -- e.g., + `list[HttpUrl]` is compared after URL normalization (trailing slash, lowercased + scheme/host) -- so it catches duplicates that differ only in normalization. The + PySpark check catches exact duplicates only. + +- `require_any_of` checks `isNotNull` as a proxy for Pydantic's `model_fields_set`. + Parquet has no equivalent of "explicitly provided"; `isNotNull` is stricter (it + rejects fields explicitly set to null). + ## Extension Points -**Adding a new output target** (Arrow schemas next, PySpark expressions after): Add a -column to `TypeMapping` in `extraction/type_registry.py` for type-name resolution. Write -a new renderer module that consumes specs and the type registry. The extraction layer and -output layout are target-independent. +**Adding a new output target**: Add a column to `TypeMapping` in +`extraction/type_registry.py` for type-name resolution. Write a pipeline module that +consumes `ModelSpec` trees and a renderer that produces output. The extraction layer is +target-independent. Register the format in `cli.py`. -**Adding a new type kind**: Add a variant to `TypeKind` in `extraction/type_analyzer.py`. +**Adding a new type kind**: Add a variant to `FieldShape` in `extraction/field.py`. Handle it in the terminal classification of `analyze_type()`. Add an extraction function -and spec dataclass if needed. Update renderers to handle the new kind. - -**Adding a new constraint type**: The iterative unwrapper collects it automatically (any -`Annotated` metadata becomes a `ConstraintSource`). Add a case to -`describe_field_constraint()` for the prose representation. +and spec dataclass if needed. Update `extraction/field_walk.py` traversal helpers and +all renderers to handle the new variant. + +**Adding a new constraint type**: `_unwrap` collects it automatically (any `Annotated` +metadata becomes a `ConstraintSource`). Add a case to +`describe_field_constraint()` for prose and to `dispatch_constraint()` for PySpark +expression mapping. + +**Adding a new PySpark check function**: Add a case in `dispatch_constraint`, +`dispatch_newtype`, or `dispatch_base_type` in `constraint_dispatch.py`. Add an +`invalid_value` case in `test_data/invalid_value.py` for test generation. The check builder and +renderer handle the new descriptor automatically. diff --git a/packages/overture-schema-codegen/docs/walkthrough.md b/packages/overture-schema-codegen/docs/walkthrough.md index 397e082f5..a755accc5 100644 --- a/packages/overture-schema-codegen/docs/walkthrough.md +++ b/packages/overture-schema-codegen/docs/walkthrough.md @@ -23,8 +23,8 @@ Documentation needs all of this. The codegen exists to preserve it. Navigating Python's type annotation machinery -- NewType chains, nested `Annotated` wrappers, union filtering, generic resolution -- is complex. The codegen does it once. -`analyze_type()` unwraps annotations into `TypeInfo`, a flat target-independent -representation. Extractors build specs from `TypeInfo`. Renderers consume specs without +`analyze_type()` unwraps annotations into `FieldShape`, a tree-shaped target-independent +representation. Extractors build specs from these shapes. Renderers consume specs without re-entering the type system. New output targets add renderers, not extraction logic. The solution decomposes into four layers. Discovery finds models. Extraction unwraps @@ -64,28 +64,15 @@ it. The entry point `overture:transportation:segment` maps to The codegen classifies these at the CLI boundary: `is_model_class` identifies concrete `BaseModel` subclasses, `is_union_alias` calls `analyze_type` to identify discriminated -unions. From that point forward both model features and union features satisfy the -`FeatureSpec` protocol and flow through the same pipeline. +unions. From that point forward both records and unions are `ModelSpec` values +(`RecordSpec | UnionSpec`) and flow through the same pipeline. ## 2. Leaf utilities -Two modules with no internal dependencies. Both serve multiple layers. - -### extraction/case_conversion.py - -Converts PascalCase to snake_case with two compiled regexes. `_ACRONYM_BOUNDARY` inserts -an underscore between an uppercase run and a capitalized word start: `HTMLParser` -becomes `HTML_Parser` becomes `html_parser`. `_CAMEL_BOUNDARY` inserts between -lowercase-or-digit and uppercase: `buildingPart` becomes `building_part`. -`to_snake_case` applies them in sequence and lowercases. - -`slug_filename` composes the conversion with a file extension. Every output file path in -the system passes through this function. - -```python ->>> slug_filename("HexColor") -'hex_color.md' -``` +One module with no internal dependencies, serving multiple layers. PascalCase to +snake_case conversion lives in `overture.schema.system.case` (used by the pyspark +generator and the markdown path assignment); markdown output filenames are +`f"{to_snake_case(name)}.md"` at the call site. ### extraction/docstring.py @@ -113,92 +100,85 @@ summaries. ## 3. Type analysis This is the module the entire package exists to house. `analyze_type` takes a raw type -annotation and returns `TypeInfo` -- a flat dataclass that fully describes the unwrapped -type without any reference to Python's typing machinery. - -### The loop - -The function runs a single `while True` loop that peels layers in fixed order. Each -iteration handles one wrapper: - -**NewType** records names at two levels. The first NewType encountered becomes -`outermost_newtype_name` (the user-facing identity, e.g. "FeatureVersion") and snapshots -the current `list_depth` into `newtype_outer_list_depth` -- capturing how many list -layers appeared before the NewType boundary. Subsequent NewTypes update -`last_newtype_name` (the innermost, used for constraint provenance and as the terminal -`base_type`). The loop unwraps via `__supertype__` and continues. - -**Annotated** collects every metadata object as a `ConstraintSource`, tagging each with -whichever NewType was most recently entered. This is how constraint provenance survives: -when `int32`'s `Annotated` layer contributes `Field(ge=0)`, the constraint records -`source="int32"`. If a `FieldInfo` carries a description, the function captures it -- -first description wins, so the outermost NewType's documentation takes precedence. - -**Union** filters out `NoneType` (marks optional), `Sentinel` instances (Pydantic's -`` marker for undeclared defaults), and `Literal` sentinel arms (like -`Literal[""]` used alongside `HttpUrl`). If multiple concrete `BaseModel` subclasses -remain after filtering, the function classifies the type as `UNION` and returns -immediately with the member tuple. Non-BaseModel multi-type unions raise -`UnsupportedUnionError`. A single remaining arm continues the loop. - -The `Literal` filtering has a guard: when a union contains *only* Literal arms (like -`Optional[Literal["x"]]`), the function keeps them rather than filtering everything out. - -**list/dict** increments `list_depth` for each `list[...]` layer (so `list[list[str]]` -records depth 2), sets dict flags, and continues into element types. Dict is the one -case where `analyze_type` recurses -- it calls itself for key and value types, storing -the results as nested `TypeInfo` objects. - -**Terminal** classification in `_classify_terminal` handles what remains after all -wrappers are peeled: `Any` becomes a PRIMITIVE, `Literal` returns with the literal value -(single-value only -- multi-value Literals get `literal_value=None`), `Enum` subclasses -become ENUM, `BaseModel` subclasses become MODEL, everything else becomes PRIMITIVE. +annotation and returns `tuple[FieldShape, bool, str | None]` -- the structural shape, +whether the field is optional, and the first description found in the annotation chain. +`FieldShape` is a discriminated union tree that fully describes the type without any +reference to Python's typing machinery. + +### The recursion + +`_unwrap` peels one annotation layer per call frame and returns a `FieldShape` subtree. +Each case handles one wrapper kind: + +**NewType** constructs a `_NewTypeCtx` carrying the NewType's name and callable +reference, then recurses into `__supertype__` with that context active. After the +recursion returns, `_erase_inner_newtypes` strips every `NewTypeShape` reachable through +the recursion result's `ArrayOf` layers so that exactly one `NewTypeShape` remains per +spine. The frame then wraps the (now wrapper-free) inner shape: +`NewTypeShape(name="FeatureVersion", inner=)`. Inner NewType names +survive as the terminal `Primitive.base_type`. + +**Annotated** collects every metadata object in the `args[1:]` slice as a +`ConstraintSource`, tagging each with the active `newtype_ctx`. If a `FieldInfo` is +present, its `metadata` list contributes additional constraint sources (Pydantic unpacks +`Field(min_length=1)` into annotated-types objects there). Descriptions are captured +from `FieldInfo.description` -- first one found wins, so the outermost annotation's +documentation takes precedence. The collected constraints are then attached to the +recursion result via `attach_constraints`, which walks any leading `NewTypeShape` +wrappers to prepend the constraints on the first structural layer (`ArrayOf`, `MapOf`, +or scalar terminal) that can hold them. Raw `MinLen` / `MaxLen` constraints are wrapped +into typed `ArrayMinLen` / `ScalarMinLen` (and `MaxLen` variants) matching the attachment +layer, so length-constraint dispatch is type-keyed downstream. + +**Union** delegates to `_peel_union`. That helper filters `NoneType` (marks optional), +`Sentinel` instances, and `Literal` sentinel arms. If multiple concrete `BaseModel` +subclasses remain, it invokes `union_resolver` and returns a `_Resolved` short-circuit. +A single remaining arm returns `_ContinueWith`, and `_unwrap` recurses into it. + +**list** recurses into the element type and wraps the result in `ArrayOf`. Nested lists +(`list[list[str]]`) produce nested `ArrayOf` instances -- there is no numeric depth +counter. Constraints contributed by an `Annotated` wrapper at any particular list level +land on that level's `ArrayOf` node because `attach_constraints` prepends to the +outermost structural layer, which is exactly the `ArrayOf` that was just constructed. + +**dict** recurses separately for key and value types (with `newtype_ctx=None` for both, +since dict keys and values are independent spines) and returns `MapOf`. + +**Terminal** classification in `_terminal` handles the base case: `Any` becomes +`AnyScalar`, `Literal` becomes `LiteralScalar`, `BaseModel` subclasses route through +`model_resolver` (or fall back to `Primitive(source_type=cls)`), everything else becomes +`Primitive(base_type=newtype_ctx.name or annotation.__name__)`. ### Concrete walkthroughs -**Segment (union path).** `analyze_type` receives the `Annotated` alias. Iteration 1 -sees `Annotated` -- collects the `FieldInfo` with discriminator metadata as a -constraint, unwraps to `Union[RoadSegment, RailSegment, WaterSegment]`. Iteration 2 sees -the union. No `None` arm, no sentinels. Three concrete `BaseModel` subclasses remain -- -the function classifies the type as `UNION` and returns immediately: `kind=UNION`, -`union_members=(RoadSegment, RailSegment, WaterSegment)`, `base_type="RoadSegment"` (the -first member). Two iterations, done. The union members are raw type objects, not -recursively analyzed -- callers that need field details call `extract_model` on each -member separately. +**Segment (union path).** `_unwrap` receives the `Annotated` alias for Segment. The +`Annotated` case collects discriminator metadata from `FieldInfo`, then sees the inner +annotation is a union. `_peel_union` finds three concrete `BaseModel` arms, invokes +`union_resolver`, and returns `_Resolved(UnionRef(...))` carrying the `UnionSpec` that +the resolver constructed. The `Annotated` handler attaches the discriminator constraints +and returns. Two frames deep, done. **FeatureVersion (NewType chain path).** `FeatureVersion = NewType("FeatureVersion", int32)` where `int32 = NewType("int32", Annotated[int, Field(ge=0, le=2147483647)])`. -Iteration 1 sees `FeatureVersion`. It's a NewType -- record -`outermost_newtype_name="FeatureVersion"`, snapshot `newtype_outer_list_depth=0` (no list -layers yet), unwrap to `int32`, continue. Iteration 2 sees -`int32`. Also a NewType -- update `last_newtype_name="int32"`, unwrap to `Annotated[int, -Field(ge=0, ...)]`, continue. Iteration 3 sees `Annotated`. Collect -`ConstraintSource(source="int32", constraint=)`, unwrap to `int`. The -loop breaks on `int` (not a NewType, not Annotated, not a union, not a container). -`_classify_terminal` returns a `TypeInfo` with `base_type="int32"`, -`newtype_name="FeatureVersion"`, `kind=PRIMITIVE`, and a constraint tuple recording the -provenance chain. - -The two paths demonstrate the function's range. Segment exits early on the union branch -with member types for downstream extraction. FeatureVersion runs the full loop through -NewType and Annotated layers, accumulating constraint provenance that survives to -rendering. - -### _UnwrapState - -The accumulator dataclass carries state across iterations: optional/dict flags, -`list_depth` (incremented per `list[...]` layer), `newtype_outer_list_depth` (snapshotted -from `list_depth` when the first NewType is entered), the constraint list, both NewType -name slots, and the captured description. Its `build_type_info` method assembles the -final `TypeInfo` from accumulated state, freezing the constraint list into a tuple. - -### walk_type_info - -A shared visitor that recurses into dict key/value `TypeInfo` children. Both type -collection and reverse reference computation use it rather than duplicating the descent -pattern. Union members are raw `type` objects (not `TypeInfo` instances), so callers -handle them directly. +Frame 1 sees `FeatureVersion` -- a NewType. Constructs `_NewTypeCtx("FeatureVersion", +FeatureVersion)`, recurses into `int32`. Frame 2 sees `int32` -- also a NewType. +Constructs `_NewTypeCtx("int32", int32)`, recurses into `Annotated[int, Field(ge=0, +...)]`. Frame 3 sees `Annotated`. Collects `ConstraintSource(source_name="int32", +constraint=)`. Recurses into `int`. Frame 4 hits the terminal +`int`. `newtype_ctx` is still `_NewTypeCtx("int32", int32)` -- frame 3 passed frame 2's +context through unchanged, since `Annotated` does not introduce a NewType -- so +`_terminal` uses `newtype_ctx.name` (`"int32"`) as `base_type`. Returns +`Primitive(base_type="int32")`. Frame 3 attaches the constraints: `Primitive` gets the +`ge=0` / `le=2147483647` sources prepended. Frame 2's `_erase_inner_newtypes` sees a +bare `Primitive` -- no `NewTypeShape` to strip -- and wraps the result in +`NewTypeShape(name="int32", inner=Primitive(...))`. Frame 1's `_erase_inner_newtypes` +strips that inner `NewTypeShape`, yielding `Primitive(...)`, and wraps it in +`NewTypeShape(name="FeatureVersion", inner=Primitive(...))`. + +The two paths demonstrate the function's range. Segment exits after two frames via +`union_resolver`. FeatureVersion recurses four frames through a NewType chain, with +constraint provenance tagging surviving to rendering. ## 4. Data structures @@ -206,45 +186,49 @@ handle them directly. a dataclass with no methods beyond field access and, in `UnionSpec`'s case, one cached property. -**FieldSpec** represents one model field: alias-resolved name, `TypeInfo`, description, -required flag. Two fields populated later by tree expansion: `model` (a reference to the -nested `ModelSpec` for MODEL-kind fields) and `starts_cycle` (true when following this -field's model would create a cycle in the ancestor chain). +**FieldSpec** represents one model field: alias-resolved name, `shape: FieldShape`, +description, required flag. `ModelRef` and `UnionRef` shapes carry their resolved specs +(populated during `extract_model` recursion), so consumers can follow the tree without a +separate expansion pass. -**ModelSpec** represents one Pydantic model: class name, cleaned docstring, fields in +**RecordSpec** represents one Pydantic model: class name, cleaned docstring, fields in documentation order, source class reference, the entry point string that located it, and model-level constraints from decorators like `@require_any_of`. **UnionSpec** represents a discriminated union type alias. Segment's `UnionSpec` carries `members=[RoadSegment, RailSegment, WaterSegment]`, `discriminator_field="subtype"`, and `common_base=TransportationSegment`. Its `annotated_fields` list pairs each `FieldSpec` -with `variant_sources` -- a tuple of class names indicating which union members -contribute that field, or `None` for fields from `TransportationSegment` shared across -all members. The `fields` cached property unwraps this for code that doesn't need -provenance. `UnionSpec` uses `eq=False` because it contains mutable lists and a -`cached_property` -- dataclass-generated `__eq__` would be unreliable. - -**FeatureSpec** is a `Protocol` satisfied by both `ModelSpec` and `UnionSpec`. This is -the pipeline's unifying abstraction. Tree expansion, type collection, rendering -dispatch, and example loading all operate on `FeatureSpec` without knowing which -concrete type they hold. +with `variant_sources` -- a tuple of `BaseModel` subclasses indicating which union +members contribute that field, or `None` for fields from `TransportationSegment` shared +across all members. The `fields` cached property unwraps this for code that doesn't need +provenance. Each member also has its already-extracted `RecordSpec` retained in +`member_specs: list[MemberSpec]` so downstream consumers (check builder, base-row +generator) reuse it instead of re-extracting the subtree. `UnionSpec` uses `eq=False` +because it contains mutable lists and a `cached_property` -- dataclass-generated +`__eq__` would be unreliable. + +**ModelSpec** is the type alias `RecordSpec | UnionSpec`. Type collection, rendering +dispatch, and example loading all operate on `ModelSpec`. Consumers narrow with +`isinstance` when they need `UnionSpec`-specific attributes like `discriminator_field`. **EnumSpec** and **EnumMemberSpec** serve enums. **NewTypeSpec** serves NewTypes. **NumericSpec** serves numeric primitives with an `Interval` for bounds and optional `float_bits`. -**SupplementarySpec** is the union type alias `EnumSpec | NewTypeSpec | ModelSpec` -- -the set of non-feature types that need their own output pages. `NumericSpec` and -geometry types are excluded because they render on aggregate pages rather than -individual ones. +**SupplementarySpec** is the union type alias `EnumSpec | NewTypeSpec | RecordSpec | +PydanticTypeSpec` -- the set of non-feature types that need their own output pages. +`PydanticTypeSpec` covers Pydantic built-ins like `HttpUrl` and `EmailStr` (carrying the +class plus a pointer back to Pydantic's docs). `NumericSpec` and geometry types are +excluded because they render on aggregate pages rather than individual ones. ### Classification functions -Three functions at the bottom of `extraction/specs.py` classify discovery results. `is_model_class` -is a `TypeGuard` that checks `isinstance(obj, type) and issubclass(obj, BaseModel)`. -`is_union_alias` calls `analyze_type` and checks for `UNION` kind -- the only place -outside the type analyzer that touches Python type annotations. `filter_model_classes` -applies the model guard across the discovery dict's values. +Three functions at the bottom of `extraction/specs.py` classify discovery results. +`is_model_class` is a `TypeGuard` that checks `isinstance(obj, type) and issubclass(obj, +BaseModel)`. `is_union_alias` calls `analyze_type` with a sentinel `union_resolver` that +raises immediately on detection -- the only place outside the type analyzer that touches +Python type annotations. `filter_model_classes` applies the model guard across the +discovery dict's values. ## 5. Type registry @@ -268,7 +252,7 @@ precedence. ## 6. Model extraction -`extract_model` converts a Pydantic `BaseModel` subclass into a `ModelSpec`. +`extract_model` converts a Pydantic `BaseModel` subclass into a `RecordSpec`. ### Field ordering @@ -291,27 +275,28 @@ classes. One subtlety: Pydantic strips the `Annotated` wrapper from some fields and moves the metadata to `field_info.metadata`. When this happens, `analyze_type` sees a bare type -and misses the constraints. `_merge_field_metadata` patches them back in, tagging them -with `source=None` since they came from the field's own annotation rather than a NewType -chain. +and misses the constraints. `_attach_field_metadata` routes them through +`attach_constraints` -- tagging them with `source=None` since they came from the field's +own annotation rather than a NewType chain -- so length-constraint typing happens here +just as it does during normal `Annotated` unwrapping. Model-level constraints come from `ModelConstraint.get_model_constraints(model_class)`, which inspects decorators like `@require_any_of` and `@require_if`. -### Tree expansion +### Recursive extraction -`expand_model_tree` is the recursive step that populates `FieldSpec.model` references. -It maintains a shared cache keyed by Python class and an ancestor set for cycle -detection. +`extract_model` recursively resolves sub-models and sub-unions during field extraction, +building `ModelRef`/`UnionRef` shapes with their specs already populated. It maintains a +shared cache keyed by Python class and an ancestor set for cycle detection. The cache insert happens *before* recursion. Without this ordering, a back-edge encounter would find no cached entry and infinite-loop instead of marking -`starts_cycle=True`. The sequence: extract the sub-model, insert it into the cache, then -recurse into its fields. Shared references (the same sub-model used in multiple fields) -reuse the cached `ModelSpec` without marking cycles. +`starts_cycle=True`. The sequence: create the partial `RecordSpec`, insert it into the +cache, then populate its fields. Shared references (the same sub-model used in multiple +fields) reuse the cached `RecordSpec` without marking cycles. -Union-kind fields skip inline expansion -- they appear as a single row in the output, -linking to their members, rather than expanding inline. +`UnionRef` fields resolve via the `union_resolver` callback -- they appear as a single +row in the output, linking to their members, rather than expanding inline. ## 7. Other extractors @@ -326,17 +311,18 @@ per-member check, so members that inherit the class docstring verbatim get ### NewType extraction `extract_newtype` calls `analyze_type` on the NewType callable and extracts the custom -docstring. When the NewType has no explicit docstring, it falls back to -`TypeInfo.description` -- the first `Field.description` found in the `Annotated` +docstring. When the NewType has no explicit docstring, it falls back to the description +returned by `analyze_type` -- the first `Field.description` found in the `Annotated` metadata chain. ### Union extraction The most involved extractor. Walk through `Segment` concretely. -`extract_union("Segment", annotation)` calls `analyze_type` on the -`Annotated[Union[RoadSegment, RailSegment, WaterSegment], ...]` alias. The analyzer -returns `kind=UNION` with the three member types. +`extract_union("Segment", annotation)` calls `_union_members`, which runs `analyze_type` +with a capturing `union_resolver` that raises out of the analysis as soon as it sees a +multi-arm union of `BaseModel` subclasses. The captured tuple gives the three member +types plus any description from enclosing `Annotated` layers. Next, `_find_common_base` intersects each member's filtered MRO (BaseModel subclasses only, excluding `BaseModel` itself). All three share `TransportationSegment` in their @@ -348,13 +334,17 @@ The extractor calls `extract_model(TransportationSegment)` to get the shared fie Fields like `id`, `geometry`, `version`, `sources`, and `subtype` appear in the common base. These become shared `AnnotatedField` entries with `variant_sources=None`. -Then it extracts each member: `RoadSegment`, `RailSegment`, `WaterSegment`. Fields not -in the shared set are variant-specific, deduplicated by `(name, type_identity)` where -`type_identity` captures `base_type`, `kind`, `is_optional`, and `list_depth`. If -`RoadSegment` and `WaterSegment` both define a `width` field with the same type -identity, the `AnnotatedField` accumulates both class names: -`variant_sources=("RoadSegment", "WaterSegment")`. Fields unique to one member get a -single-element tuple. +Then it extracts each member: `RoadSegment`, `RailSegment`, `WaterSegment`. Each result +is retained on the `UnionSpec` as a `MemberSpec(member_cls, spec)` so consumers don't +re-extract. Fields not in the shared set are variant-specific, deduplicated by +`(name, structural_fingerprint)` where the fingerprint walks the field's `FieldShape` +tree, capturing every wrapper layer plus the terminal type. If `RoadSegment` and +`WaterSegment` both define a `width` field with the same fingerprint, the +`AnnotatedField` accumulates both classes: `variant_sources=(RoadSegment, +WaterSegment)`. Fields unique to one member get a single-element tuple. When two members +declare the same field name with the same structural fingerprint but diverging +constraints, the extractor raises rather than silently dropping one member's +constraints. `extract_discriminator` inspects the `Annotated` metadata for a `FieldInfo` with a discriminator attribute. For Segment, it finds `subtype` and builds the mapping: @@ -435,24 +425,23 @@ discover every referenced type that needs its own output page: enums, semantic N and sub-models. The walk maintains a visited set for models and a feature name set for skip detection. -Types that are themselves top-level features get skipped. For UNION-kind fields, the -function extracts and walks each member's fields. For semantic NewTypes, it walks the -`__supertype__` chain to collect intermediate NewTypes -- `Id` wraps -`NoWhitespaceString` wraps `str`, and both `Id` and `NoWhitespaceString` get their own -pages. The `walk_type_info` visitor handles dict key/value recursion. +Types that are themselves top-level features get skipped. For `UnionRef` fields, the function extracts and walks each member's fields. For +semantic NewTypes, it walks the `__supertype__` chain to collect intermediate NewTypes -- +`Id` wraps `NoWhitespaceString` wraps `str`, and both `Id` and `NoWhitespaceString` get +their own pages. `walk_shape` from `field_walk.py` handles recursion into `ArrayOf`, +`MapOf`, and `NewTypeShape` wrappers. -MODEL-kind fields follow `field_spec.model` references that were populated by -`expand_model_tree`. The function raises `RuntimeError` if it encounters a MODEL-kind -field with `model=None` -- a guard against calling collection before tree expansion. +`ModelRef` fields follow their `.model` reference (populated during `extract_model` +recursion) into nested `RecordSpec` trees. A single field matches multiple conditions independently. A semantic NewType wrapping a -MODEL-kind type triggers both NewType extraction and model collection. The checks use +`ModelRef` triggers both NewType extraction and model collection. The checks use independent `if` statements, not `elif`. ## 11. Path assignment -`build_placement_registry` builds the complete mapping from type names to output file -paths. Three tiers: +`build_placement_registry` builds the complete `dict[TypeIdentity, PurePosixPath]` +mapping each type to its output file path. Four tiers: Aggregate pages come first. All numeric primitives point to `system/primitive/primitives.md`. All geometry types point to @@ -460,7 +449,8 @@ Aggregate pages come first. All numeric primitives point to reference page. Feature specs get individual pages. Output directories derive from -`output_dir_for_entry_point`. Filenames use `slug_filename`. +`output_dir_for_entry_point`. Filenames are the snake-case type name with a `.md` +extension. Supplementary specs get module-derived paths from `source_type.__module__`. When a supplementary type's output directory falls under a feature directory, @@ -472,15 +462,20 @@ cluttering feature directories. `_nest_under_types` sorts feature directories by path length (descending) before checking containment, so the most specific match wins. +`PydanticTypeSpec` entries (e.g. `HttpUrl`) bypass module mirroring and land at +`pydantic//.md`, keeping the generated Pydantic reference set +isolated from theme directories. + ## 12. Links and reverse references ### Link computation -`LinkContext` carries the current page's output path and the full type-to-path registry. -When a renderer formats a type reference, it calls `resolve_link` to compute a relative -path from the current page to the target. Types without registry entries return `None`, -telling renderers to show inline code instead of a broken link. `resolve_link_or_slug` -provides a fallback when a link is required regardless. +`LinkContext` carries the current page's output path and the full `dict[TypeIdentity, +PurePosixPath]` registry. When a renderer formats a type reference, it calls +`resolve_link` with the target's `TypeIdentity` to compute a relative path. Identities +without registry entries return `None`, telling renderers to show inline code instead +of a broken link. `resolve_link_or_slug` provides a fallback when a link is required +regardless. `relative_link` computes `../` navigation between any two paths in the output tree. It finds the common prefix of directory components, counts the levels up from the source @@ -490,8 +485,9 @@ rejects `..` components to prevent path traversal surprises. ### Reverse references `compute_reverse_references` walks all feature fields and supplementary specs to build -`dict[str, list[UsedByEntry]]`. Each entry maps a type name to the list of types that -reference it. Entries sort models before NewTypes, alphabetical within each group. +`dict[TypeIdentity, list[UsedByEntry]]`. Each entry maps a target identity to the list +of types that reference it. Entries sort models before NewTypes, alphabetical within +each group. The function tracks references with sets for deduplication, then sorts into lists at the end. It skips self-references and references to types not in the supplementary spec dict @@ -504,29 +500,28 @@ provenance rather than direct field reference. ## 13. Markdown type formatting -`markdown/type_format.py` converts `TypeInfo` into display strings for markdown output. +`markdown/type_format.py` converts a field's `FieldShape` into display strings for +markdown output. -`format_type` handles the full range of field types. Single-value Literals render as -`"value"` in backticks. Semantic NewTypes and enums/models get markdown links via -`_resolve_type_link`, which checks the `LinkContext` registry and falls back to plain +`format_type` handles the full range of field types. Single-value `LiteralScalar`s +render as `"value"` in backticks. Semantic NewTypes and enums/models get markdown links +via `_resolve_type_link`, which checks the `LinkContext` registry and falls back to plain code spans. For types with a linked identity (semantic NewTypes, enums, models), list -rendering depends on where the list layers sit relative to the NewType boundary. -`newtype_outer_list_depth > 0` means the list wraps the NewType (`list[PhoneNumber]`) and -renders as `list`. `is_list` with `newtype_name` set means the NewType -wraps a list internally (`Sources` wrapping `list[SourceItem]`) and renders with a -`(list)` qualifier. Non-NewType identities (enums, models) use `list` syntax. Linked -inner types use broken-backtick syntax (`` `list<` `` ... `` `>` ``) built as a single -wrapper to avoid adjacent backticks that CommonMark would interpret as multi-backtick -code span delimiters. Dict types render as `` `map` ``. Qualifiers (optional, list, -map) append in parentheses. - -Union members format independently -- each gets its own link resolution, joined with -pipe separators escaped for table-cell safety. +rendering depends on where the `ArrayOf` layers sit relative to the `NewTypeShape` +boundary. An `ArrayOf` sitting outside the `NewTypeShape` in the shape tree means the +list wraps the NewType (`list[PhoneNumber]`) and renders as `list`. A +`NewTypeShape` with an `ArrayOf` inner means the NewType wraps a list internally +(`Sources` wrapping `list[SourceItem]`) and renders with a `(list)` qualifier. Non-NewType +identities (enums, models) use `list` syntax. Linked inner types use broken-backtick +syntax (`` `list<` `` ... `` `>` ``) built as a single wrapper to avoid adjacent backticks +that CommonMark would interpret as multi-backtick code span delimiters. `MapOf` shapes +render as `` `map` ``. Qualifiers (optional, list, map) append in parentheses. + +`UnionRef` members format independently -- each gets its own link resolution, joined +with pipe separators escaped for table-cell safety. `format_underlying_type` handles NewType page headers. It links enums and models that -have their own pages but skips the outermost NewType name to avoid self-referencing. The -function uses `source_type.__name__` rather than `base_type` for link resolution, since -`base_type` may carry the outermost NewType name when only one NewType wraps a class. +have their own pages but skips the outermost NewType name to avoid self-referencing. ## 14. Markdown rendering @@ -548,7 +543,7 @@ spans. ### Field expansion -`render_feature` dispatches on spec type. `ModelSpec` gets `_expand_model_fields`, which +`render_model` dispatches on spec type. `RecordSpec` gets `_expand_model_fields`, which walks the pre-populated `FieldSpec.model` tree and produces dot-notation rows. `sources[0].dataset` appears as a single row in the flat field table, with `[]` appended per nesting level to list-of-model fields (so a doubly-nested list gets @@ -631,29 +626,26 @@ pipeline. `generate_markdown_pages` in `markdown/pipeline.py` is the "main" function. It takes feature specs and a schema root, returns rendered pages without touching the filesystem. -Eight steps: - -1. **Expand model trees** with a shared cache across all features, so sub-models - referenced by multiple features extract once. +Seven steps (tree expansion now happens inside `extract_model`): -2. **Partition primitive and geometry names** from the system primitive module's +1. **Partition primitive and geometry names** from the system primitive module's `__all__` exports. -3. **Collect supplementary types** by walking expanded feature trees. +2. **Collect supplementary types** by walking feature trees. -4. **Build the placement registry** mapping every type to its output file path. +3. **Build the placement registry** mapping every type to its output file path. -5. **Compute reverse references** across all features and supplements. +4. **Compute reverse references** across all features and supplements. -6. **Render each feature** with its `LinkContext`, loaded examples, and used-by entries. +5. **Render each feature** with its `LinkContext`, loaded examples, and used-by entries. -7. **Render each supplementary type** -- dispatching to `render_enum`, `render_newtype`, - or `render_feature` (for sub-models) based on spec type. +6. **Render each supplementary type** -- dispatching to `render_enum`, `render_newtype`, + `render_model` (for sub-models), or `render_pydantic_type` based on spec type. -8. **Render aggregate pages** for primitives and geometry. +7. **Render aggregate pages** for primitives and geometry. The return value is `list[RenderedPage]` -- frozen dataclasses carrying content, output -path, and a boolean `is_feature` flag. The caller decides what to do with them. +path, and a boolean `is_model` flag. The caller decides what to do with them. ### The CLI @@ -688,36 +680,34 @@ A reader who reached this point has seen every module in isolation. This section entry_point="overture.schema.transportation:Segment")`. **Classification.** The CLI tests each entry. `is_model_class(Segment)` returns false -- -`Segment` is not a class. `is_union_alias(Segment)` calls `analyze_type`, which peels -the `Annotated` wrapper and finds three `BaseModel` subclasses in the union. The -analyzer returns `kind=UNION`. The CLI routes Segment to `extract_union`. - -**Extraction.** `extract_union("Segment", annotation)` calls `analyze_type` again (cheap --- the same two-iteration path), gets the three member types, and finds -`TransportationSegment` as the common base via `_find_common_base`. It extracts the -common base's fields as shared, then extracts each member's fields and partitions the -non-shared ones into `AnnotatedField` entries with variant provenance. +`Segment` is not a class. `is_union_alias(Segment)` calls `analyze_type` with a sentinel +`union_resolver` that raises on detection. The CLI routes Segment to `extract_union`. + +**Extraction.** `extract_union("Segment", annotation)` calls `_union_members`, which +runs `analyze_type` with a capturing `union_resolver` to grab the three member types +plus the union description. `_find_common_base` picks `TransportationSegment` as the +shared parent. The extractor calls `extract_model` on the common base and on each +member -- the results are cached on the `UnionSpec` as `member_specs` -- and partitions +the non-shared fields into `AnnotatedField` entries with variant provenance. `extract_discriminator` finds `subtype` and builds `{"road": RoadSegment, "rail": -RailSegment, "water": WaterSegment}`. The result is a `UnionSpec` satisfying -`FeatureSpec`. +RailSegment, "water": WaterSegment}`. The result is a `UnionSpec` (a `ModelSpec`). Meanwhile, concrete models like `Building` go through `extract_model`, which calls `analyze_type` on each field annotation. A field typed `FeatureVersion` unwraps through -two NewType layers and an `Annotated` layer, producing a `TypeInfo` with -`base_type="int32"`, `newtype_name="FeatureVersion"`, and constraint provenance linking -`ge=0` back to the `int32` NewType. Both extraction paths produce specs satisfying -`FeatureSpec`. +two NewType layers and an `Annotated` layer, producing a `NewTypeShape(name="FeatureVersion", +inner=Primitive(base_type="int32", constraints=(...)))` shape with constraint provenance +linking `ge=0` back to the `int32` NewType. Both extraction paths produce `ModelSpec` +values. **Pipeline entry.** The feature specs enter `generate_markdown_pages`. -`expand_model_tree` walks MODEL-kind fields on Segment's `UnionSpec` and populates -`FieldSpec.model` references. The shared cache ensures sub-models referenced by multiple -features (like `Sources`) extract once. Union-kind fields skip inline expansion. +Sub-model `FieldShape` trees are fully resolved -- `ModelRef` nodes already carry their +`RecordSpec` from recursive `extract_model` calls. No separate expansion pass is needed. **Layout.** `partition_numeric_and_geometry_types` reads the system module's exports. -`collect_all_supplementary_types` walks Segment's expanded fields and discovers -referenced enums (like `Subtype`), semantic NewTypes (like `Id`, `Sources`), and -sub-models. The walk follows `FieldSpec.model` references down the tree, and for -UNION-kind fields, extracts and walks each member's fields separately. +`collect_all_supplementary_types` walks Segment's field shapes and discovers referenced +enums (like `Subtype`), semantic NewTypes (like `Id`, `Sources`), and sub-models. The +walk follows `ModelRef.model` references down the tree, and for `UnionRef` shapes, +extracts and walks each member's fields separately. `build_placement_registry` assigns Segment's output path from its entry point: `entry_point_module` extracts `overture.schema.transportation`, `compute_output_dir` @@ -729,13 +719,13 @@ that Segment references `Subtype`, `Id`, `Sources`, and other types. These refer populate "Used By" sections: the `Subtype` enum page shows that Segment uses it. **Rendering.** The pipeline builds a `LinkContext` from Segment's output path and the -full registry. `render_feature` dispatches to `_expand_union_fields` because the spec is +full registry. `render_model` dispatches to `_expand_union_fields` because the spec is a `UnionSpec`. Shared fields from `TransportationSegment` render as plain rows. Variant-specific fields get italic tags: `` `road_class` *(Road)* ``. The renderer -formats each field's type via `format_type`, which resolves links through the +formats each field's `FieldShape` via `format_type`, which resolves links through the `LinkContext` -- `Subtype` gets a relative link to its enum page, `Id` links to its -NewType page. Constraints with `source=None` annotate field rows; constraints with named -sources appear on the source NewType's page instead. +NewType page. Constraints with `source_name=None` annotate field rows; constraints with +named sources appear on the source NewType's page instead. The example loader finds `pyproject.toml` in the transportation theme package, reads `[examples.Segment]`, validates each example against the union alias (injecting literal @@ -747,7 +737,7 @@ The Jinja2 template assembles the field table, optional constraints section, exa and "Used By" partial into markdown. **Output.** The pipeline returns a `RenderedPage` with Segment's content, its output -path, and `is_feature=True`. The CLI prepends Docusaurus frontmatter and writes the +path, and `is_model=True`. The CLI prepends Docusaurus frontmatter and writes the file. `_category_.json` files get generated for sidebar navigation. **The layering principle.** At every stage, the modules that do the work never reach diff --git a/packages/overture-schema-codegen/pyproject.toml b/packages/overture-schema-codegen/pyproject.toml index 3019a6a92..c3ee37847 100644 --- a/packages/overture-schema-codegen/pyproject.toml +++ b/packages/overture-schema-codegen/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "overture-schema-common", "overture-schema-system", "tomli>=2.0; python_version < '3.11'", + "typing-extensions>=4.0", ] description = "Code generator that produces documentation and code from Pydantic models" dynamic = ["version"] @@ -20,9 +21,25 @@ name = "overture-schema-codegen" overture-codegen = "overture.schema.codegen.cli:main" [tool.uv.sources] +overture-schema-addresses-theme = { workspace = true } +overture-schema-base-theme = { workspace = true } +overture-schema-buildings-theme = { workspace = true } overture-schema-cli = { workspace = true } overture-schema-common = { workspace = true } +overture-schema-divisions-theme = { workspace = true } +overture-schema-places-theme = { workspace = true } overture-schema-system = { workspace = true } +overture-schema-transportation-theme = { workspace = true } + +[dependency-groups] +test = [ + "overture-schema-addresses-theme", + "overture-schema-base-theme", + "overture-schema-buildings-theme", + "overture-schema-divisions-theme", + "overture-schema-places-theme", + "overture-schema-transportation-theme", +] [tool.hatch.version] path = "src/overture/schema/codegen/__about__.py" diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py index 279f22a84..667843692 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/cli.py @@ -12,26 +12,21 @@ filter_models, ) -from .extraction.model_extraction import extract_model -from .extraction.specs import ( - FeatureSpec, - is_model_class, - is_union_alias, -) -from .extraction.union_extraction import extract_union +from .extraction.specs import ModelSpec from .layout.module_layout import ( OUTPUT_ROOT, compute_schema_root, - entry_point_class, entry_point_module, ) from .markdown.pipeline import generate_markdown_pages +from .pyspark.pipeline import generate_pyspark_modules +from .spec_discovery import extract_model_spec log = logging.getLogger(__name__) __all__ = ["cli"] -_OUTPUT_FORMATS = ("markdown",) +_OUTPUT_FORMATS = ("markdown", "pyspark") _FEATURE_FRONTMATTER = "---\nsidebar_position: 1\n---\n\n" @@ -84,7 +79,15 @@ def list_models() -> None: "--output-dir", type=click.Path(path_type=Path), default=None, - help="Write output to directory (default: stdout)", + help="Write output files directly into this directory (default: stdout). " + "For pyspark, writes expression modules (*.py). " + "For markdown, writes theme subdirectories.", +) +@click.option( + "--test-output-dir", + type=click.Path(path_type=Path), + default=None, + help="Write test modules (test_*.py) into this directory (pyspark only).", ) def generate( output_format: str, @@ -92,55 +95,71 @@ def generate( filters: tuple[str, ...], excludes: tuple[str, ...], output_dir: Path | None, + test_output_dir: Path | None, ) -> None: """Generate code/docs from discovered models.""" - all_models = discover_models() + if output_format != "pyspark" and test_output_dir is not None: + raise click.UsageError("--test-output-dir is only valid with --format pyspark") - # Schema root from ALL entry points (before tag filters). - module_paths = [entry_point_module(k.entry_point) for k in all_models] - schema_root = compute_schema_root(module_paths) + all_models = discover_models() models = filter_models(all_models, build_selector(tags, filters, excludes)) if output_dir: output_dir.mkdir(parents=True, exist_ok=True) - feature_specs: list[FeatureSpec] = [] - for key, entry in models.items(): - if is_model_class(entry): - feature_specs.append(extract_model(entry, entry_point=key.entry_point)) - elif is_union_alias(entry): - feature_specs.append( - extract_union( - entry_point_class(key.entry_point), - entry, - entry_point=key.entry_point, - ) - ) + model_specs: list[ModelSpec] = [ + spec + for key, entry in models.items() + if (spec := extract_model_spec(key, entry)) is not None + ] - _generate_markdown(feature_specs, schema_root, output_dir) + if output_format == "pyspark": + _generate_pyspark(model_specs, output_dir, test_output_dir) + else: + module_paths = [entry_point_module(k.entry_point) for k in all_models] + schema_root = compute_schema_root(module_paths) + _generate_markdown(model_specs, schema_root, output_dir) def _generate_markdown( - feature_specs: list[FeatureSpec], + model_specs: list[ModelSpec], schema_root: str, output_dir: Path | None, ) -> None: """Generate markdown with directory layout and placement-aware links.""" - pages = generate_markdown_pages(feature_specs, schema_root) + pages = generate_markdown_pages(model_specs, schema_root) for page in pages: content = ( - f"{_FEATURE_FRONTMATTER}{page.content}" if page.is_feature else page.content + f"{_FEATURE_FRONTMATTER}{page.content}" if page.is_model else page.content ) _write_output(content, output_dir, page.path) if output_dir: - feature_paths = {page.path for page in pages if page.is_feature} + feature_paths = {page.path for page in pages if page.is_model} all_paths = {page.path for page in pages} _write_category_files(output_dir, all_paths, feature_paths) +def _generate_pyspark( + model_specs: list[ModelSpec], + output_dir: Path | None, + test_output_dir: Path | None = None, +) -> None: + """Generate PySpark validation modules. + + Output is syntactically valid Python; we assume a code formatter runs + over the written directories afterwards to match existing conventions. + """ + modules = generate_pyspark_modules(model_specs) + for mod in modules.source: + _write_output(mod.content, output_dir, mod.path) + if test_output_dir is not None: + for mod in modules.test: + _write_output(mod.content, test_output_dir, mod.path) + + def _ancestor_dirs(paths: set[PurePosixPath]) -> set[PurePosixPath]: """Collect all ancestor directories for a set of file paths.""" dirs: set[PurePosixPath] = set() diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/case_conversion.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/case_conversion.py deleted file mode 100644 index 9d06341fb..000000000 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/case_conversion.py +++ /dev/null @@ -1,41 +0,0 @@ -"""PascalCase to snake_case conversion for code generation.""" - -import re - -__all__ = ["slug_filename", "to_snake_case"] - -# Insert _ between an acronym run and a capitalized word start (HTML|Parser) -_ACRONYM_BOUNDARY = re.compile(r"([A-Z]+)([A-Z][a-z])") -# Insert _ between a lowercase/digit and an uppercase letter (building|Part) -_CAMEL_BOUNDARY = re.compile(r"([a-z0-9])([A-Z])") - - -def to_snake_case(name: str) -> str: - """Convert PascalCase to snake_case. - - Handles acronym runs correctly: "HTMLParser" becomes "html_parser", - not "h_t_m_l_parser". - - >>> to_snake_case("HTMLParser") - 'html_parser' - >>> to_snake_case("BuildingPart") - 'building_part' - >>> to_snake_case("simple") - 'simple' - """ - name = _ACRONYM_BOUNDARY.sub(r"\1_\2", name) - name = _CAMEL_BOUNDARY.sub(r"\1_\2", name) - return name.lower() - - -def slug_filename(name: str, ext: str = ".md") -> str: - """Convert a PascalCase type name to a snake_case filename. - - >>> slug_filename("HexColor") - 'hex_color.md' - >>> slug_filename("BuildingPart") - 'building_part.md' - >>> slug_filename("BuildingPart", ext=".json") - 'building_part.json' - """ - return f"{to_snake_case(name)}{ext}" diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/field.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/field.py new file mode 100644 index 000000000..3fc37f4fb --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/field.py @@ -0,0 +1,172 @@ +"""Tree-shaped IR for model field types. + +`FieldShape` is a discriminated union -- `Primitive`, `LiteralScalar`, +`AnyScalar`, `ModelRef`, `UnionRef`, `ArrayOf`, `MapOf`, `NewTypeShape` +-- nested to describe arbitrary list / dict / NewType wrapping. Each +variant carries its own constraints (where meaningful), and walkers +encounter each constraint at the layer it targets. + +The three terminal scalar variants (`Primitive`, `LiteralScalar`, +`AnyScalar`) are grouped under the `Scalar` type alias for consumers +that only need to ask "is this a leaf?". + +`NewTypeShape` wraps an inner shape, so its position relative to +`ArrayOf` is structural: `NewTypeShape(inner=ArrayOf(...))` is a +NewType over `list[X]`, while `ArrayOf(element=NewTypeShape(...))` +is a list of NewType-wrapped values. Consumers pattern-match on +shape to distinguish the two. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypeAlias + +if TYPE_CHECKING: + from .specs import RecordSpec, UnionSpec + +__all__ = [ + "AnyScalar", + "ArrayOf", + "ConstraintSource", + "FieldShape", + "LiteralScalar", + "MapOf", + "ModelRef", + "NewTypeShape", + "Primitive", + "Scalar", + "UnionRef", +] + + +@dataclass(frozen=True, slots=True) +class ConstraintSource: + """A constraint paired with the NewType that contributed it. + + `source_ref` and `source_name` identify the NewType that declared + the constraint; both are `None` for constraints contributed directly + on a field annotation rather than through a NewType. `constraint` + is the raw metadata object from `Annotated[..., constraint]`. + """ + + source_ref: object | None + source_name: str | None + constraint: object + + +@dataclass(frozen=True, slots=True) +class Primitive: + """Terminal type with a registry lookup key. + + Covers primitives (`int32`, `str`), enums, Pydantic built-ins + (`HttpUrl`, `EmailStr`), and `BaseModel` subclasses that weren't + resolved to a `ModelRef` (e.g. when no `model_resolver` was + supplied). + """ + + base_type: str + source_type: type | None = None + constraints: tuple[ConstraintSource, ...] = () + + +@dataclass(frozen=True, slots=True) +class LiteralScalar: + """`Literal[X, ...]` terminal.""" + + values: tuple[object, ...] + constraints: tuple[ConstraintSource, ...] = () + + +@dataclass(frozen=True, slots=True) +class AnyScalar: + """`typing.Any` terminal.""" + + constraints: tuple[ConstraintSource, ...] = () + + +Scalar: TypeAlias = Primitive | LiteralScalar | AnyScalar +"""Terminal shape: a value that doesn't wrap another shape. + +Consumers that just need "is this a leaf?" check `isinstance(x, Scalar)`; +consumers that need terminal-specific data narrow to a variant. +""" + + +@dataclass(frozen=True, slots=True) +class ModelRef: + """Reference to a Pydantic sub-model. + + `starts_cycle` marks the back-edge of a cycle in the model graph; + consumers that recurse into models must stop at cycle starts. + """ + + model: RecordSpec + starts_cycle: bool = False + + +@dataclass(frozen=True, slots=True) +class UnionRef: + """Reference to a discriminated union of models.""" + + union: UnionSpec + + +@dataclass(frozen=True, slots=True) +class ArrayOf: + """Sequence of values sharing a single element shape. + + Nested arrays are nested `ArrayOf` instances; there is no numeric + depth field. `constraints` carries array-level validation rules + (length, uniqueness). Per-element constraints live on `element` + and its descendants. + """ + + element: FieldShape + constraints: tuple[ConstraintSource, ...] = () + + +@dataclass(frozen=True, slots=True) +class MapOf: + """Mapping from a key shape to a value shape. + + `constraints` carries map-level validation rules. Per-key and + per-value constraints live on `key` / `value` respectively. + """ + + key: FieldShape + value: FieldShape + constraints: tuple[ConstraintSource, ...] = () + + +@dataclass(frozen=True, slots=True) +class NewTypeShape: + """A NewType wrapper around an inner shape. + + Position relative to other wrappers is meaningful: + `NewTypeShape(inner=ArrayOf(...))` is a NewType over `list[X]`; + `ArrayOf(element=NewTypeShape(...))` is a list of NewType-wrapped + values. Consumers distinguish the two by pattern, not a numeric + offset. + + Constraints contributed by the NewType chain attach to the + `Scalar` / `ArrayOf` / `MapOf` layer they target, not to the + wrapper itself. `name` and `ref` identify the NewType for linking + without owning constraint state. + """ + + name: str + ref: object + inner: FieldShape + + +FieldShape: TypeAlias = ( + Primitive + | LiteralScalar + | AnyScalar + | ModelRef + | UnionRef + | ArrayOf + | MapOf + | NewTypeShape +) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/field_constraints.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/field_constraints.py index 0db927065..c62f4adbd 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/field_constraints.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/field_constraints.py @@ -1,37 +1,33 @@ """Convert field-level constraints to display text. Handles constraints from Annotated metadata and NewType wrappers: -Ge, Gt, Interval, Le, Lt, MaxLen, MinLen, GeometryTypeConstraint, -Reference, and custom constraint classes. +Ge, Gt, Interval, Le, Lt, ArrayMinLen, ArrayMaxLen, ScalarMinLen, +ScalarMaxLen, GeometryTypeConstraint, Reference, and custom constraint +classes. """ from __future__ import annotations +import re from collections.abc import Callable -from annotated_types import Ge, Gt, Interval, Le, Lt, MaxLen, MinLen +from annotated_types import Ge, Gt, Interval, Le, Lt from overture.schema.system.primitive import GeometryTypeConstraint from overture.schema.system.ref import Reference from .docstring import first_docstring_line +from .length_constraints import ArrayMaxLen, ArrayMinLen, ScalarMaxLen, ScalarMinLen +from .literal_alternatives import LiteralAlternatives from .specs import TypeIdentity from .type_analyzer import ConstraintSource __all__ = [ "constraint_display_text", - "constraint_pattern", "describe_field_constraint", ] -# Bound attribute names paired with display operators. Each entry maps an -# annotated_types constraint attribute (Ge, Gt, Le, Lt, Interval) to its -# mathematical symbol for prose rendering. -# -# numeric_extraction.py has its own _BOUND_ATTRS for numeric extraction. The -# duplication is deliberate: these modules use the same attribute names for -# unrelated purposes (display formatting vs. numeric bound extraction), and -# coupling them for four string literals adds a dependency without value. +# Bound attribute -> mathematical symbol for prose rendering. _BOUND_OPS: tuple[tuple[str, str], ...] = ( ("ge", "≥"), ("gt", ">"), @@ -108,10 +104,12 @@ def describe_field_constraint( result = _first_bound(constraint) if result is not None: return result - if isinstance(constraint, MinLen): + if isinstance(constraint, (ArrayMinLen, ScalarMinLen)): return f"Minimum length: {constraint.min_length}" - if isinstance(constraint, MaxLen): + if isinstance(constraint, (ArrayMaxLen, ScalarMaxLen)): return f"Maximum length: {constraint.max_length}" + if isinstance(constraint, LiteralAlternatives): + return "Also accepts: " + ", ".join(f"`{v!r}`" for v in constraint.values) if _is_opaque_constraint(constraint): return f"`{type(constraint).__name__}`" @@ -130,14 +128,41 @@ def _constraint_class_description(constraint: object) -> str | None: return line or None -def constraint_pattern(constraint: object) -> str | None: - """Extract the regex pattern string from a constraint, if present. +# re.UNICODE is the implicit default on compiled `str` patterns; rendering it +# would stamp a noise `(?u)` group onto every pattern. Every other flag with a +# visible matching effect is surfaced in the documented pattern. Unlike the +# pyspark dispatch (`compiled_pattern_source`) -- which must reject flags +# Spark's rlike cannot honor -- display is faithful for known flags and never +# fails: a flag absent from this table is dropped from the rendered group, not +# raised on. A new flag added to pyspark's supported set with a visible effect +# belongs here too, or docs will hide that pattern's real behavior. +_DISPLAY_FLAG_LETTERS: tuple[tuple[re.RegexFlag, str], ...] = ( + (re.IGNORECASE, "i"), + (re.MULTILINE, "m"), + (re.DOTALL, "s"), + (re.VERBOSE, "x"), + (re.ASCII, "a"), +) + + +def _inline_flag_prefix(flags: int) -> str: + """Render set regex flags as an inline group like `(?im)`, or "" if none.""" + letters = "".join(c for flag, c in _DISPLAY_FLAG_LETTERS if flags & flag) + return f"(?{letters})" if letters else "" - Traverses two levels: constraint.pattern is a compiled re.Pattern - object, and re.Pattern.pattern is the raw string. + +def _constraint_pattern(constraint: object) -> str | None: + """Return a constraint's compiled regex as displayable source, or None. + + Prepends an inline-flag group (e.g. `(?i)` for case-insensitivity) so a + flagged pattern reads as the regex that actually matches rather than its + bare, misleading source. Returns None when `constraint.pattern` is not a + compiled `re.Pattern`. """ compiled = getattr(constraint, "pattern", None) - return getattr(compiled, "pattern", None) + if not isinstance(compiled, re.Pattern): + return None + return f"{_inline_flag_prefix(compiled.flags)}{compiled.pattern}" def constraint_display_text( @@ -148,7 +173,7 @@ def constraint_display_text( description = _constraint_class_description(cs.constraint) if _is_opaque_constraint(cs.constraint) and description: cls_name = type(cs.constraint).__name__ - pattern = constraint_pattern(cs.constraint) + pattern = _constraint_pattern(cs.constraint) if pattern: return f"{description} (`{cls_name}`, pattern: `{pattern}`)" return f"{description} (`{cls_name}`)" diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/field_walk.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/field_walk.py new file mode 100644 index 000000000..7384c61c2 --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/field_walk.py @@ -0,0 +1,275 @@ +"""Generic traversal helpers over `FieldShape` trees. + +`shape_children` (one-level child enumeration) and `walk_shape` +(pre-order DFS) cover open-ended traversals; `terminal_of`, +`terminal_scalar`, `list_depth`, `newtype_name`, and `all_constraints` +cover the most common derived views. `ModelRef` and `UnionRef` are +leaves -- the walker does not cross model or union boundaries +automatically; that's a per-consumer decision. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from enum import Enum + +from typing_extensions import assert_never + +from .field import ( + AnyScalar, + ArrayOf, + ConstraintSource, + FieldShape, + LiteralScalar, + MapOf, + ModelRef, + NewTypeShape, + Primitive, + Scalar, + UnionRef, +) + +__all__ = [ + "all_constraints", + "enum_source", + "has_array_layer", + "list_depth", + "map_key_value_constraints", + "newtype_name", + "shape_children", + "terminal_model_ref", + "terminal_of", + "terminal_primitive", + "terminal_scalar", + "terminal_union_ref", + "walk_shape", +] + + +def terminal_of(shape: FieldShape) -> FieldShape: + """Unwrap `ArrayOf` and `NewTypeShape` layers to find the terminal shape. + + Returns the innermost shape that isn't a sequence or NewType wrapper. + `Scalar`, `ModelRef`, `UnionRef`, and `MapOf` count as terminals. + """ + while True: + match shape: + case ArrayOf(element=inner) | NewTypeShape(inner=inner): + shape = inner + case ( + Primitive() + | LiteralScalar() + | AnyScalar() + | ModelRef() + | UnionRef() + | MapOf() + ): + return shape + case _: + assert_never(shape) + + +def terminal_scalar(shape: FieldShape) -> Scalar | None: + """Return the terminal `Scalar`, or `None` for non-scalar terminals.""" + terminal = terminal_of(shape) + return terminal if isinstance(terminal, Scalar) else None + + +def terminal_primitive(shape: FieldShape) -> Primitive | None: + """Return the terminal `Primitive`, or `None` for non-primitive terminals. + + Like `terminal_scalar`, but returns `None` for `LiteralScalar` and + `AnyScalar` — use this when the caller needs `base_type` or + `source_type`, which only exist on `Primitive`. + """ + terminal = terminal_of(shape) + return terminal if isinstance(terminal, Primitive) else None + + +def terminal_model_ref(shape: FieldShape) -> ModelRef | None: + """Return the terminal `ModelRef`, or `None` for non-model terminals.""" + terminal = terminal_of(shape) + return terminal if isinstance(terminal, ModelRef) else None + + +def terminal_union_ref(shape: FieldShape) -> UnionRef | None: + """Return the terminal `UnionRef`, or `None` for non-union terminals.""" + terminal = terminal_of(shape) + return terminal if isinstance(terminal, UnionRef) else None + + +def enum_source(shape: FieldShape) -> type[Enum] | None: + """Return the `Enum` class backing a `Primitive`, or `None`. + + Returns the `Enum` subclass stored in `Primitive.source_type` when + `shape` is a `Primitive` and `source_type` is an `Enum` subclass. + Returns `None` for every other shape, including wrappers: a + `NewTypeShape` wrapping an enum-backed `Primitive` returns `None`, + not the inner enum. + + Parameters + ---------- + shape + The shape to inspect. + + Returns + ------- + type[Enum] or None + The `Enum` class when `shape` is a `Primitive` backed by one, + `None` otherwise. + """ + if not isinstance(shape, Primitive): + return None + src = shape.source_type + if isinstance(src, type) and issubclass(src, Enum): + return src + return None + + +def shape_children(shape: FieldShape) -> Iterator[FieldShape]: + """Yield direct child shapes within *shape* (one level deep). + + `Scalar`, `ModelRef`, and `UnionRef` have no children. + """ + match shape: + case ArrayOf(element=element): + yield element + case MapOf(key=key, value=value): + yield key + yield value + case NewTypeShape(inner=inner): + yield inner + case Primitive() | LiteralScalar() | AnyScalar() | ModelRef() | UnionRef(): + return + case _: + assert_never(shape) + + +def walk_shape(shape: FieldShape, visit: Callable[[FieldShape], None]) -> None: + """Pre-order traversal of a `FieldShape` tree. + + Visits *shape*, then descends into each direct child via + `shape_children`. Stops at `ModelRef` / `UnionRef` -- recursion + across model boundaries is the caller's choice. + """ + visit(shape) + for child in shape_children(shape): + walk_shape(child, visit) + + +def list_depth(shape: FieldShape) -> int: + """Total number of `ArrayOf` layers in *shape*, looking through `NewTypeShape`. + + A NewType wrapping a list counts the same as a list wrapping a + NewType. + """ + depth = 0 + cur = shape + while True: + match cur: + case ArrayOf(element=element): + depth += 1 + cur = element + case NewTypeShape(inner=inner): + cur = inner + case ( + Primitive() + | LiteralScalar() + | AnyScalar() + | ModelRef() + | UnionRef() + | MapOf() + ): + return depth + case _: + assert_never(cur) + + +def has_array_layer(shape: FieldShape) -> bool: + """Whether *shape* has any `ArrayOf` layer, looking through `NewTypeShape`. + + Prefer this over `list_depth(shape) > 0` -- callers that only need + "is this array-shaped" don't need to count layers. + """ + cur = shape + while isinstance(cur, NewTypeShape): + cur = cur.inner + return isinstance(cur, ArrayOf) + + +def newtype_name(shape: FieldShape) -> str | None: + """Return the outermost `NewTypeShape` name, looking through `ArrayOf` layers.""" + cur: FieldShape = shape + while isinstance(cur, ArrayOf): + cur = cur.element + match cur: + case NewTypeShape(name=name): + return name + case ( + Primitive() + | LiteralScalar() + | AnyScalar() + | ModelRef() + | UnionRef() + | MapOf() + ): + return None + case _: + assert_never(cur) + + +def all_constraints(shape: FieldShape) -> tuple[ConstraintSource, ...]: + """Concatenate the field's own constraints from every layer of *shape*. + + Walks `NewTypeShape` and `ArrayOf` wrappers to gather constraints + that apply to this field. Stops at `MapOf` (key/value constraints + belong to nested key/value shapes, not to the enclosing field) and + at `ModelRef` / `UnionRef` (which carry no constraints). Constraints + from outer `ArrayOf` layers appear before constraints from inner + layers, matching the structural order of the shape tree. + """ + collected: list[ConstraintSource] = [] + cur = shape + while True: + match cur: + case ArrayOf(element=inner, constraints=cs): + collected.extend(cs) + cur = inner + case NewTypeShape(inner=inner): + cur = inner + case ( + Primitive(constraints=cs) + | LiteralScalar(constraints=cs) + | AnyScalar(constraints=cs) + ): + collected.extend(cs) + return tuple(collected) + case MapOf(constraints=cs): + collected.extend(cs) + return tuple(collected) + case ModelRef() | UnionRef(): + return tuple(collected) + case _: + assert_never(cur) + + +def map_key_value_constraints( + shape: FieldShape, +) -> tuple[tuple[ConstraintSource, ...], tuple[ConstraintSource, ...]]: + """Return a `MapOf` terminal's (key_constraints, value_constraints), or `((), ())`. + + Looks through `NewTypeShape` / `ArrayOf` wrappers to find a `MapOf`, + then gathers each side's constraints with `all_constraints`. This + surfaces per-key and per-value rules that `all_constraints` on the + enclosing field deliberately stops short of (it treats `MapOf` as a + terminal). Returns `((), ())` when *shape* has no `MapOf` terminal. + """ + cur = shape + while True: + match cur: + case NewTypeShape(inner=inner) | ArrayOf(element=inner): + cur = inner + case MapOf(key=key, value=value): + return all_constraints(key), all_constraints(value) + case _: + return (), () diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/length_constraints.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/length_constraints.py new file mode 100644 index 000000000..0d8635efa --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/length_constraints.py @@ -0,0 +1,49 @@ +"""Internal typed length-constraint classes. + +`annotated_types.MaxLen` and `annotated_types.MinLen` are polysemous: +`MaxLen(10)` on a `str` constrains character count, while `MaxLen(10)` +on a `list[X]` constrains cardinality. The codegen extractor splits +them by attachment layer so each variant carries its own dispatch: +`ArrayMinLen` / `ArrayMaxLen` for `ArrayOf` layers, `ScalarMinLen` / +`ScalarMaxLen` for scalar layers. + +These are codegen-internal classes -- schema authors continue to write +the normal Pydantic form (`Field(min_length=n)` / `Field(max_length=n)`), +which Pydantic lowers into the `annotated_types.MinLen` / `MaxLen` +metadata described above. The wrapping into these layer-typed variants +happens inside `type_analyzer.attach_constraints` when the constraint +reaches its target layer. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from annotated_types import MaxLen, MinLen + +__all__ = [ + "ArrayMaxLen", + "ArrayMinLen", + "ScalarMaxLen", + "ScalarMinLen", +] + + +@dataclass(frozen=True) +class ArrayMinLen(MinLen): + """Cardinality lower bound for an `ArrayOf` layer.""" + + +@dataclass(frozen=True) +class ArrayMaxLen(MaxLen): + """Cardinality upper bound for an `ArrayOf` layer.""" + + +@dataclass(frozen=True) +class ScalarMinLen(MinLen): + """Character-count lower bound for a scalar layer.""" + + +@dataclass(frozen=True) +class ScalarMaxLen(MaxLen): + """Character-count upper bound for a scalar layer.""" diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/literal_alternatives.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/literal_alternatives.py new file mode 100644 index 000000000..5d48d1059 --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/literal_alternatives.py @@ -0,0 +1,26 @@ +"""Internal constraint recording a union's literal alternatives. + +A field annotated `X | Literal[c, ...]` validates as "the concrete arm `X`'s +checks pass OR the value is one of `c, ...`". `type_analyzer._peel_union` keeps +the concrete arm as the field's shape (so downstream consumers still see a +`Primitive` / `NewTypeShape` rather than a union of scalar-and-literal) and +records the dropped literal values in this constraint on that shape's layer. + +Consumers read it to let those literal values bypass the concrete arm's +constraints: the PySpark dispatch emits a value-exact bypass, and the markdown +renderer notes the accepted literals. Codegen-internal -- schema authors write +the plain `X | Literal[c]` union; nothing constructs this class directly. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +__all__ = ["LiteralAlternatives"] + + +@dataclass(frozen=True, slots=True) +class LiteralAlternatives: + """Literal values a union field accepts alongside its concrete arm.""" + + values: tuple[object, ...] diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/model_extraction.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/model_extraction.py index 76807e123..1f4a28167 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/model_extraction.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/model_extraction.py @@ -1,8 +1,8 @@ -"""Model extraction and tree expansion.""" +"""Pydantic model extraction into `RecordSpec`.""" from __future__ import annotations -import dataclasses +from collections.abc import Mapping from pydantic import BaseModel from pydantic.fields import FieldInfo @@ -11,11 +11,22 @@ from overture.schema.system.model_constraint import ModelConstraint from .docstring import clean_docstring -from .specs import FeatureSpec, FieldSpec, ModelSpec, is_model_class -from .type_analyzer import ConstraintSource, TypeInfo, TypeKind, analyze_type +from .field import ( + ConstraintSource, + FieldShape, + ModelRef, + UnionRef, +) +from .specs import FieldSpec, RecordSpec, is_model_class +from .type_analyzer import ( + ModelResolver, + UnionResolver, + analyze_type, + attach_constraints, + unwrap_list, +) __all__ = [ - "expand_model_tree", "extract_model", "resolve_field_alias", ] @@ -37,28 +48,30 @@ def resolve_field_alias(field_name: str, field_info: FieldInfo) -> str: return field_name -def _merge_field_metadata(type_info: TypeInfo, field_info: FieldInfo) -> TypeInfo: - """Merge constraints from field_info.metadata into TypeInfo. - - Pydantic strips the Annotated wrapper from some fields (non-optional, - non-union) and moves the metadata to field_info.metadata. When this - happens, analyze_type sees a bare type and misses the constraints. - The two sets never overlap: field_info.metadata is empty when the - Annotated wrapper survives in the annotation. - """ - if not field_info.metadata: - return type_info - extra = tuple(ConstraintSource(None, None, m) for m in field_info.metadata) - return dataclasses.replace(type_info, constraints=type_info.constraints + extra) - - -def _is_field_required(field_info: FieldInfo, type_info: TypeInfo) -> bool: +def _is_field_required(field_info: FieldInfo, is_optional: bool) -> bool: """Determine whether a field is required (no default and not Optional).""" has_default = ( field_info.default is not PydanticUndefined or field_info.default_factory is not None ) - return not has_default and not type_info.is_optional + return not has_default and not is_optional + + +def _attach_field_metadata(shape: FieldShape, field_info: FieldInfo) -> FieldShape: + """Merge constraints from `field_info.metadata` onto *shape*. + + Pydantic strips the outermost Annotated wrapper from some fields + (non-optional, non-union) and moves its metadata to + `field_info.metadata`. When that happens `analyze_type` sees a bare + type and misses those constraints. They anchor at the topmost + constraint-bearing layer, so we route them through + `attach_constraints` so that length-constraint wrapping applies here + just as it does during normal annotation unwrapping. + """ + if not field_info.metadata: + return shape + extra = tuple(ConstraintSource(None, None, m) for m in field_info.metadata) + return attach_constraints(shape, extra) def _basemodel_bases(cls: type) -> list[type[BaseModel]]: @@ -88,13 +101,13 @@ def _class_order(model_class: type[BaseModel]) -> list[type]: def _field_order(model_class: type[BaseModel]) -> list[str]: - """Return model_fields keys in documentation order. + """Return `model_fields` keys in documentation order. Walks the class hierarchy recursively. At each level of multiple - inheritance, the first base is the "primary chain" and the rest - are "mixins." Primary chain and own fields come first, then mixin - fields in declaration order. Single-inheritance levels use - Pydantic's default reversed-MRO order. + inheritance, the first base is the primary chain and the rest are + mixins. Primary chain and own fields come first, then mixin fields + in declaration order. Single-inheritance levels use Pydantic's + default reversed-MRO order. """ valid_names = set(model_class.model_fields.keys()) result: list[str] = [] @@ -111,94 +124,125 @@ def extract_model( model_class: type[BaseModel], *, entry_point: str | None = None, -) -> ModelSpec: - """Extract model specification from a Pydantic model class.""" - field_info_map = model_class.model_fields - ordered_keys = _field_order(model_class) - - fields: list[FieldSpec] = [] - for field_name in ordered_keys: - field_info = field_info_map[field_name] - output_name = resolve_field_alias(field_name, field_info) - - # Use field_info.annotation (resolved TypeVars) not get_type_hints - annotation = field_info.annotation - if annotation is None: - continue + partitions: Mapping[str, str] | None = None, +) -> RecordSpec: + """Extract a fully-resolved `RecordSpec` from a Pydantic model class. + + Recurses into sub-models and unions, producing `ModelRef` / + `UnionRef` terminals with their specs resolved. Cycles in the + model graph (a field whose source type is an ancestor on the + current extraction stack) produce a `ModelRef` pointing at the + in-progress ancestor spec with `starts_cycle=True` so consumers + stop recursion at the back-edge. + """ + return _extract_model_recursive( + model_class, + entry_point=entry_point, + partitions=partitions or {}, + cache={}, + ancestors=frozenset(), + ) - type_info = _merge_field_metadata(analyze_type(annotation), field_info) - fields.append( - FieldSpec( - name=output_name, - type_info=type_info, - description=field_info.description or type_info.description, - is_required=_is_field_required(field_info, type_info), - ) - ) +def _extract_model_recursive( + model_class: type[BaseModel], + *, + entry_point: str | None, + partitions: Mapping[str, str], + cache: dict[type, RecordSpec], + ancestors: frozenset[type], +) -> RecordSpec: + """Inner recursive helper for `extract_model`. - return ModelSpec( + Inserts the (partial) `RecordSpec` into `cache` before populating + its fields so cycles can find it. `ancestors` is the set of types + currently on the recursion stack -- a sub-field whose source type + appears there is a back-edge and gets `starts_cycle=True`. + """ + spec = RecordSpec( name=model_class.__name__, description=clean_docstring(model_class.__doc__), - fields=fields, + fields=[], source_type=model_class, entry_point=entry_point, + partitions=partitions, constraints=ModelConstraint.get_model_constraints(model_class), ) + cache[model_class] = spec + descendant_ancestors = ancestors | {model_class} + model_resolver, union_resolver = _make_resolvers(cache, descendant_ancestors) -def expand_model_tree( - spec: FeatureSpec, - cache: dict[type, ModelSpec] | None = None, -) -> FeatureSpec: - """Populate model references on MODEL-kind fields, recursively. - - Walks *spec*'s fields and sets `field.model` for fields whose type - is a Pydantic model. Uses *cache* to reuse already-extracted ModelSpecs - and detect shared references. Marks fields whose model creates a cycle - in the ancestor chain with `starts_cycle=True`. + fields: list[FieldSpec] = [] + for field_name in _field_order(model_class): + field_info = model_class.model_fields[field_name] + annotation = field_info.annotation + if annotation is None: + continue + shape, is_optional, ti_description = analyze_type( + annotation, + owner=model_class, + model_resolver=model_resolver, + union_resolver=union_resolver, + ) + shape = _attach_field_metadata(shape, field_info) + fields.append( + FieldSpec( + name=resolve_field_alias(field_name, field_info), + shape=shape, + description=field_info.description or ti_description, + is_required=_is_field_required(field_info, is_optional), + is_optional=is_optional, + ) + ) - Mutates *spec* in place and returns it. - """ - if cache is None: - cache = {} - if isinstance(spec, ModelSpec) and spec.source_type is not None: - cache[spec.source_type] = spec - ancestors = frozenset({spec.source_type}) if spec.source_type else frozenset() - _expand_fields(spec.fields, cache, ancestors) + spec.fields = fields return spec -def _expand_fields( - fields: list[FieldSpec], - cache: dict[type, ModelSpec], +def _make_resolvers( + cache: dict[type, RecordSpec], ancestors: frozenset[type], -) -> None: - """Recursive helper for expand_model_tree. - - Cache insertion happens before recursion — cycle detection depends - on the ancestor's ModelSpec being in the cache when the back-edge - is encountered. +) -> tuple[ModelResolver, UnionResolver]: + """Build the resolvers that recursively extract sub-models / sub-unions. + + `cache` shares already-extracted sub-specs across a single + extraction so sub-models referenced more than once share a + `RecordSpec`. `ancestors` carries the recursion stack for cycle + detection -- a back-edge produces a `ModelRef` pointing at the + in-progress ancestor spec with `starts_cycle=True`. """ - for field_spec in fields: - ti = field_spec.type_info - source = ti.source_type - if ti.kind == TypeKind.UNION: - # Union fields have no single model to recurse into. - # The field row appears in the output; skip inline expansion. - continue - if ti.kind != TypeKind.MODEL or source is None: - continue - if source in ancestors: - # Cycle: reuse existing spec, mark the edge - field_spec.model = cache.get(source) - field_spec.starts_cycle = True - elif source in cache: - # Shared reference: reuse, not a cycle - field_spec.model = cache[source] - else: - sub_spec = extract_model(source) - cache[source] = sub_spec # insert BEFORE recursing - field_spec.model = sub_spec - _expand_fields(sub_spec.fields, cache, ancestors | {source}) + def resolve_model(cls: type[BaseModel]) -> ModelRef: + if cls in ancestors: + return ModelRef(model=cache[cls], starts_cycle=True) + cached = cache.get(cls) + if cached is not None: + return ModelRef(model=cached) + sub_spec = _extract_model_recursive( + cls, + entry_point=None, + partitions={}, + cache=cache, + ancestors=ancestors, + ) + return ModelRef(model=sub_spec) + + def resolve_union( + annotation: object, + members: tuple[type[BaseModel], ...], + _description: str | None, + ) -> UnionRef: + # Late import: extract_union calls back into extract_model for + # member classes. A module-level import would be a cycle. + from .union_extraction import extract_union # noqa: PLC0415 + + # Recover the union alias name: `analyze_type` reaches the + # union via `members[0].__name__` when the alias name is lost + # (plain `Foo = Annotated[...]` doesn't preserve it pre-PEP-695). + # Convention: members extend `Base`. + placeholder = members[0].__name__ if members else "" + sub_union = extract_union(placeholder, unwrap_list(annotation)) + return UnionRef(union=sub_union) + + return resolve_model, resolve_union diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/newtype_extraction.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/newtype_extraction.py index ff11c770a..5e074d259 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/newtype_extraction.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/newtype_extraction.py @@ -1,6 +1,7 @@ """NewType extraction.""" from .docstring import clean_docstring, is_custom_docstring +from .field import NewTypeShape from .specs import NewTypeSpec from .type_analyzer import analyze_type @@ -8,19 +9,31 @@ def extract_newtype(newtype_callable: object) -> NewTypeSpec: - """Extract NewType specification from a NewType callable.""" - type_info = analyze_type(newtype_callable) - doc = getattr(newtype_callable, "__doc__", None) - name = type_info.newtype_name or getattr(newtype_callable, "__name__", None) + """Extract a `NewTypeSpec` from a NewType callable. + + `analyze_type(newtype_callable)` returns a shape whose outermost + layer is the NewType's own `NewTypeShape`. We strip that wrapper so + `NewTypeSpec.shape` describes the *underlying* type -- the NewType + isn't a self-reference on its own page. + """ + shape, _, ti_description = analyze_type(newtype_callable) + + name = getattr(newtype_callable, "__name__", None) + if isinstance(shape, NewTypeShape) and shape.name == name: + underlying = shape.inner + else: + underlying = shape + if name is None: msg = f"Cannot determine name for NewType: {newtype_callable!r}" raise ValueError(msg) - description = ( - clean_docstring(doc) if is_custom_docstring(doc) else type_info.description - ) + + doc = getattr(newtype_callable, "__doc__", None) + description = clean_docstring(doc) if is_custom_docstring(doc) else ti_description + return NewTypeSpec( name=name, description=description, - type_info=type_info, + shape=underlying, source_type=newtype_callable, ) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/numeric_extraction.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/numeric_extraction.py index ae899a4e6..7416d42f8 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/numeric_extraction.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/numeric_extraction.py @@ -3,9 +3,10 @@ from annotated_types import Interval from .docstring import first_docstring_line +from .field import FieldShape, Scalar +from .field_walk import terminal_of from .newtype_extraction import extract_newtype from .specs import NumericSpec, TypeIdentity -from .type_analyzer import TypeInfo __all__ = [ "extract_numeric_bounds", @@ -13,24 +14,22 @@ ] -# Bound attribute names on annotated_types constraint objects (Ge, Gt, Le, -# Lt, Interval) used for numeric bound extraction. -# -# field_constraints.py has its own _BOUND_OPS for display formatting. -# The duplication is deliberate: these modules use the same attribute names -# for unrelated purposes (numeric extraction vs. prose rendering), and -# coupling them for four string literals adds a dependency without value. +# Bound attribute names on annotated_types constraints (Ge, Gt, Le, Lt, Interval). _BOUND_ATTRS = ("ge", "gt", "le", "lt") -def extract_numeric_bounds(type_info: TypeInfo) -> Interval: - """Extract numeric bounds from a TypeInfo's constraints. +def extract_numeric_bounds(shape: FieldShape) -> Interval: + """Extract numeric bounds from the constraints on a shape's terminal scalar. - Checks for ge, gt, le, and lt attributes on constraint objects. - Stops at the first constraint defining each bound. + Walks `NewTypeShape` / `ArrayOf` wrappers to find the terminal + `Scalar`, then scans its constraints for `ge`, `gt`, `le`, and `lt` + attributes. Stops at the first constraint defining each bound. """ + terminal = terminal_of(shape) + if not isinstance(terminal, Scalar): + return Interval() found: dict[str, int | float] = {} - for cs in type_info.constraints: + for cs in terminal.constraints: c = cs.constraint for attr in _BOUND_ATTRS: if attr not in found: @@ -47,7 +46,10 @@ def extract_numerics( specs: list[NumericSpec] = [] for tid in numeric_ids: newtype_spec = extract_newtype(tid.obj) - bounds = extract_numeric_bounds(newtype_spec.type_info) + # extract_newtype strips the outer NewTypeShape, so the spec's + # terminal scalar already carries the constraints the NewType + # contributed -- extract_numeric_bounds walks straight to it. + bounds = extract_numeric_bounds(newtype_spec.shape) description = first_docstring_line(getattr(tid.obj, "__doc__", None)) float_bits = _extract_float_bits(tid.name) specs.append( @@ -68,5 +70,5 @@ def extract_numerics( def _extract_float_bits(name: str) -> int | None: - """Extract bit width from a float type name like 'float32'.""" + """Extract bit width from a float type name like `float32`.""" return _FLOAT_BITS.get(name) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/specs.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/specs.py index acba1577d..880e07d9a 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/specs.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/specs.py @@ -3,23 +3,27 @@ from __future__ import annotations import functools +from collections.abc import Mapping from dataclasses import dataclass, field -from typing import Any, Protocol, TypeGuard, runtime_checkable +from typing import Any, TypeAlias, TypeGuard from annotated_types import Interval from pydantic import BaseModel +from overture.schema.system.discovery.tag import get_values_for_key from overture.schema.system.model_constraint import ModelConstraint -from .type_analyzer import TypeInfo, TypeKind, UnsupportedUnionError, analyze_type +from .field import FieldShape +from .type_analyzer import capture_union_members __all__ = [ "AnnotatedField", "EnumMemberSpec", "EnumSpec", - "FeatureSpec", - "FieldSpec", "ModelSpec", + "FieldSpec", + "MemberSpec", + "RecordSpec", "NewTypeSpec", "NumericSpec", "PydanticTypeSpec", @@ -28,11 +32,22 @@ "filter_model_classes", "is_model_class", "is_pydantic_sourced", - "is_pydantic_type", "is_union_alias", + "partitions_from_tags", ] +def partitions_from_tags(tags: frozenset[str]) -> dict[str, str]: + """Map registry tags to Hive partition columns for a feature. + + Today populated only from `overture:theme=`; the value object is + a generic name -> value map so additional partition keys (e.g. release + version) can be added without changing the surrounding pipeline. + """ + theme = next(iter(get_values_for_key(tags, "overture:theme")), None) + return {"theme": theme} if theme is not None else {} + + @dataclass(frozen=True, eq=False) class TypeIdentity: """Unique identity for a type in the codegen system. @@ -69,7 +84,7 @@ def module(self) -> str: class _SourceTypeIdentityMixin: """Mixin providing `identity` from `source_type` and `name`. - Shared by EnumSpec, ModelSpec, NewTypeSpec, and PydanticTypeSpec -- + Shared by EnumSpec, RecordSpec, NewTypeSpec, and PydanticTypeSpec -- each has a `source_type` (the Python class/callable) and a `name`. UnionSpec uses `source_annotation` instead, so it defines its own `identity`. @@ -106,35 +121,22 @@ class EnumSpec(_SourceTypeIdentityMixin): @dataclass class FieldSpec: - """Specification for a model field.""" + """Specification for a model field: header metadata plus structural shape. - name: str - type_info: TypeInfo - description: str | None - is_required: bool - model: ModelSpec | None = None - starts_cycle: bool = False - - -@runtime_checkable -class FeatureSpec(Protocol): - """Shared interface for feature-level specs (ModelSpec, UnionSpec).""" + `shape` is the full `FieldShape` tree, including any sub-model + (`ModelRef`) and sub-union (`UnionRef`) references already + resolved during extraction. + """ name: str - description: str | None - source_type: type[BaseModel] | None - entry_point: str | None - constraints: tuple[ModelConstraint, ...] - - @property - def fields(self) -> list[FieldSpec]: ... - - @property - def identity(self) -> TypeIdentity: ... + shape: FieldShape + description: str | None = None + is_required: bool = True + is_optional: bool = False @dataclass -class ModelSpec(_SourceTypeIdentityMixin): +class RecordSpec(_SourceTypeIdentityMixin): """Specification for a Pydantic model.""" name: str @@ -142,6 +144,7 @@ class ModelSpec(_SourceTypeIdentityMixin): fields: list[FieldSpec] = field(default_factory=list) source_type: type[BaseModel] | None = None entry_point: str | None = None + partitions: Mapping[str, str] = field(default_factory=dict) constraints: tuple[ModelConstraint, ...] = () @@ -150,11 +153,26 @@ class AnnotatedField: """A FieldSpec paired with union variant provenance.""" field_spec: FieldSpec - variant_sources: tuple[str, ...] | None + variant_sources: tuple[type[BaseModel], ...] | None + + +@dataclass +class MemberSpec: + """A union member's class paired with its extracted `RecordSpec`. + + `extract_union` already runs `extract_model` on every member to + build the merged `annotated_fields`; retaining the result here lets + consumers (check builder, base-row generator) reuse it instead of + re-extracting the same subtree. + """ + + member_cls: type[BaseModel] + spec: RecordSpec -# eq=False: contains mutable lists and a cached_property, so -# dataclass-generated __eq__ would be unreliable. +# eq=False: contains mutable lists and a cached_property, so the +# dataclass-generated __eq__ would compare by value over mutable fields and +# __hash__ would be disabled (unhashable). Consumers key on object identity. @dataclass(eq=False) class UnionSpec: """Specification for a discriminated union type alias.""" @@ -167,8 +185,10 @@ class UnionSpec: discriminator_mapping: dict[str, type[BaseModel]] | None source_annotation: object common_base: type[BaseModel] + member_specs: list[MemberSpec] = field(default_factory=list) source_type: type[BaseModel] | None = field(default=None, init=False) entry_point: str | None = None + partitions: Mapping[str, str] = field(default_factory=dict) constraints: tuple[ModelConstraint, ...] = () @functools.cached_property @@ -183,11 +203,16 @@ def identity(self) -> TypeIdentity: @dataclass class NewTypeSpec(_SourceTypeIdentityMixin): - """Specification for a NewType.""" + """Specification for a NewType. + + `shape` is the underlying shape -- i.e. the `inner` of the + NewType's own `NewTypeShape` wrapper, with the wrapper stripped + so the NewType isn't a self-reference on its own page. + """ name: str description: str | None - type_info: TypeInfo + shape: FieldShape source_type: object | None = None @@ -219,8 +244,16 @@ def docs_url(self) -> str: ) -SupplementarySpec = EnumSpec | NewTypeSpec | ModelSpec | PydanticTypeSpec -"""Non-feature types referenced by feature models. +ModelSpec: TypeAlias = RecordSpec | UnionSpec +"""A model is one record, or a tagged union of records. + +The top-level type passed through the extraction pipeline. Consumers +narrow with `isinstance` when an arm-specific attribute is needed +(e.g. `UnionSpec.discriminator_field`). +""" + +SupplementarySpec = EnumSpec | NewTypeSpec | RecordSpec | PydanticTypeSpec +"""Supplementary types referenced by models. Excludes NumericSpec and geometry types, which are extracted separately via dedicated functions. @@ -232,15 +265,6 @@ def is_pydantic_sourced(source_type: type | None) -> bool: return getattr(source_type, "__module__", "").startswith("pydantic") -def is_pydantic_type(ti: TypeInfo) -> bool: - """Check whether a TypeInfo represents a Pydantic built-in type.""" - return ( - ti.kind == TypeKind.PRIMITIVE - and ti.source_type is not None - and is_pydantic_sourced(ti.source_type) - ) - - def is_model_class(obj: object) -> TypeGuard[type[BaseModel]]: """Check whether *obj* is a concrete BaseModel subclass (not a type alias).""" return isinstance(obj, type) and issubclass(obj, BaseModel) @@ -248,11 +272,7 @@ def is_model_class(obj: object) -> TypeGuard[type[BaseModel]]: def is_union_alias(obj: object) -> bool: """Check whether *obj* is a discriminated union type alias of BaseModel subclasses.""" - try: - ti = analyze_type(obj) - except (TypeError, UnsupportedUnionError): - return False - return ti.kind == TypeKind.UNION + return capture_union_members(obj) is not None def filter_model_classes(models: dict[Any, Any]) -> list[type[BaseModel]]: diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/type_analyzer.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/type_analyzer.py index a0cd5314f..7c3c6a71d 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/type_analyzer.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/type_analyzer.py @@ -1,344 +1,643 @@ -"""Iterative type unwrapping for Pydantic model annotations.""" +"""Annotation-to-`FieldShape` analysis. + +`analyze_type` recurses through a Python type annotation, peeling +`NewType`, `Annotated`, `Optional`, `list`, and `dict` layers one frame +at a time, and produces a `FieldShape` describing the structure with +constraints attached to the layer they target. + +Forward references encountered along the way are resolved against the +`owner` model's namespace before classification. Builtin generics store +`list["Node"]`'s element as a bare `str` (not a `ForwardRef`), which +neither Pydantic nor `typing.get_type_hints` resolves; resolving it here +lets a self-referential field reach its model terminal so the cycle +guard in `extract_model` engages. + +Each `Annotated` frame attaches its metadata to the shape its inner +annotation unwraps to, so that, e.g., the inner and outer `MinLen` in +`Annotated[list[Annotated[str, MinLen(2)]], MinLen(3)]` land on +different layers as different typed variants: `ArrayMinLen(3)` on the +`ArrayOf`, `ScalarMinLen(2)` on the `Primitive`. + +MODEL and UNION terminals are resolved via optional callbacks. When +no resolver is supplied a MODEL terminal falls back to +`Primitive(source_type=cls)`; a multi-arm UNION raises +`UnsupportedUnionError`. Callers that need to recurse into sub-models +pass resolvers that build a `ModelRef`/`UnionRef` with the resolved +spec. +""" from __future__ import annotations import types from collections.abc import Callable -from dataclasses import dataclass, field -from enum import Enum, auto -from typing import Annotated, Any, Literal, Union, get_args, get_origin - +from dataclasses import dataclass, replace +from typing import ( + Annotated, + Any, + ForwardRef, + Literal, + NoReturn, + Union, + get_args, + get_origin, +) + +from annotated_types import MaxLen, MinLen from pydantic import BaseModel from pydantic.fields import FieldInfo -from typing_extensions import Sentinel +from typing_extensions import Sentinel, assert_never, evaluate_forward_ref from .docstring import clean_docstring +from .field import ( + AnyScalar, + ArrayOf, + ConstraintSource, + FieldShape, + LiteralScalar, + MapOf, + ModelRef, + NewTypeShape, + Primitive, + UnionRef, +) +from .field_walk import terminal_of +from .length_constraints import ArrayMaxLen, ArrayMinLen, ScalarMaxLen, ScalarMinLen +from .literal_alternatives import LiteralAlternatives + + +@dataclass(frozen=True, slots=True) +class _ContinueWith: + """`_peel_union` result: next annotation to keep peeling. + + `literal_alternatives` carries the values of any `Literal[...]` arms + dropped in favor of a single concrete arm, so the caller can attach a + `LiteralAlternatives` constraint to the recursed shape. + """ + + annotation: object + is_optional: bool + literal_alternatives: tuple[object, ...] = () + + +@dataclass(frozen=True, slots=True) +class _Resolved: + """`_peel_union` result: finished shape, short-circuit the unwrap.""" + + shape: FieldShape + is_optional: bool + + +@dataclass(frozen=True, slots=True) +class _NewTypeCtx: + """The innermost NewType currently in scope.""" + + name: str + ref: object + __all__ = [ "ConstraintSource", - "TypeKind", - "TypeInfo", + "ModelResolver", + "UnionResolver", + "UnresolvedForwardRefError", "UnsupportedUnionError", "analyze_type", + "attach_constraints", + "capture_union_members", "is_newtype", "single_literal_value", - "walk_type_info", + "unwrap_list", ] class UnsupportedUnionError(TypeError): - """Raised when analyze_type encounters a multi-type union it cannot represent.""" + """Raised when `analyze_type` encounters a multi-type union it cannot represent.""" -class TypeKind(Enum): - """Classification of type kinds.""" +class UnresolvedForwardRefError(TypeError): + """Raised when a forward-reference annotation cannot be resolved to a type. - PRIMITIVE = auto() - LITERAL = auto() - ENUM = auto() - MODEL = auto() - UNION = auto() + Subclasses `TypeError` so callers that already guard `analyze_type` + with `except (TypeError, UnsupportedUnionError)` treat an unresolvable + forward ref as the analyzable-shape failure it is. + """ -@dataclass(slots=True) -class ConstraintSource: - """A constraint paired with the NewType that contributed it.""" +ModelResolver = Callable[[type[BaseModel]], FieldShape] +"""Resolver invoked when `analyze_type` reaches a `BaseModel` terminal.""" - source_ref: object | None - source_name: str | None - constraint: object +UnionResolver = Callable[[object, tuple[type[BaseModel], ...], str | None], FieldShape] +"""Resolver invoked at a multi-arm union terminal. +Receives the original union annotation, the tuple of member classes, +and the description accumulated from enclosing `Annotated` layers. +""" -@dataclass(slots=True) -class TypeInfo: - """Information about a type annotation.""" - base_type: str - kind: TypeKind - is_optional: bool = False - list_depth: int = 0 - newtype_outer_list_depth: int = 0 - is_dict: bool = False - dict_key_type: TypeInfo | None = None - dict_value_type: TypeInfo | None = None - constraints: tuple[ConstraintSource, ...] = () - literal_values: tuple[object, ...] | None = None - source_type: type | None = None - newtype_name: str | None = None - newtype_ref: object | None = None - union_members: tuple[type[BaseModel], ...] | None = None - description: str | None = None +def is_newtype(annotation: object) -> bool: + """Check whether *annotation* is a `typing.NewType`. - @property - def is_list(self) -> bool: - """Whether this type has any list wrapping.""" - return self.list_depth > 0 + NewType creates a callable with a `__supertype__` attribute pointing + to the wrapped type. No public API exists for this check. + """ + return callable(annotation) and hasattr(annotation, "__supertype__") -def walk_type_info(ti: TypeInfo, visitor: Callable[[TypeInfo], None]) -> None: - """Call *visitor* on *ti*, then recurse into dict key/value types. +class _UnionCaptured(Exception): # noqa: N818 - control flow, not a true error + """Raised by the capturing union resolver to short-circuit analyze_type.""" - Captures the shared recursive descent pattern used by type collection - and reverse reference computation. Union members are `type` objects - (not `TypeInfo`), so callers handle them directly. - """ - visitor(ti) - if ti.dict_key_type is not None: - walk_type_info(ti.dict_key_type, visitor) - if ti.dict_value_type is not None: - walk_type_info(ti.dict_value_type, visitor) + def __init__( + self, members: tuple[type[BaseModel], ...], description: str | None + ) -> None: + self.members = members + self.description = description -def is_newtype(annotation: object) -> bool: - """Check if annotation is a typing.NewType. +def capture_union_members( + annotation: object, +) -> tuple[tuple[type[BaseModel], ...], str | None] | None: + """Peel wrappers from *annotation* and return its union members. - NewType creates a callable with a __supertype__ attribute pointing - to the wrapped type. No public API exists for this check. + Returns `(members, description)` when *annotation* (possibly wrapped + in `Annotated`) terminates in a multi-arm union of `BaseModel` + subclasses, otherwise `None`. Internally drives `analyze_type` with + a capturing resolver and unwinds via an exception once the union + terminal is reached. The resolver fires only after every enclosing + `Annotated` layer is peeled, so the captured description matches what + `analyze_type` would return. """ - return callable(annotation) and hasattr(annotation, "__supertype__") + + def _capture( + _ann: object, + members: tuple[type[BaseModel], ...], + description: str | None, + ) -> NoReturn: + raise _UnionCaptured(members, description) + + try: + analyze_type(annotation, union_resolver=_capture) + except _UnionCaptured as captured: + return captured.members, captured.description + except (TypeError, UnsupportedUnionError): + return None + return None def _is_union(origin: object) -> bool: - """Check if an origin represents a union type (X | Y or Union[X, Y]).""" + """Whether an origin represents a union type (`X | Y` or `Union[X, Y]`).""" return origin in (types.UnionType, Union) -@dataclass(slots=True) -class _UnwrapState: - """Accumulated state from iterative type unwrapping. +def _filter_sentinel_arms(args: tuple[object, ...]) -> list[object]: + """Remove `NoneType` and `Sentinel` arms from union type arguments.""" + return [a for a in args if a is not types.NoneType and not isinstance(a, Sentinel)] + + +def analyze_type( + annotation: object, + *, + owner: type | None = None, + model_resolver: ModelResolver | None = None, + union_resolver: UnionResolver | None = None, +) -> tuple[FieldShape, bool, str | None]: + """Analyze an annotation into a `FieldShape` plus field-level metadata. + + Parameters + ---------- + annotation + The annotation to analyze. + owner + The model class these annotations belong to. Supplies the + namespace for resolving forward references (`list["Node"]` + stores `"Node"` as a bare string). When None, an unresolvable + forward ref raises `UnresolvedForwardRefError`. + model_resolver + Optional callback invoked when the terminal is a `BaseModel` + subclass. Returns the `FieldShape` to use at that position -- + typically a `ModelRef` with a resolved `RecordSpec`. Defaults to + a `Scalar` carrying the class as `source_type` for callers that + cannot resolve sub-models (e.g. dict key/value analysis). + union_resolver + Optional callback invoked when the terminal is a multi-arm + union of `BaseModel` subclasses. Returns the `FieldShape` to + use -- typically a `UnionRef` with a resolved `UnionSpec`. + Required to support unions; raises otherwise. + + Returns + ------- + tuple[FieldShape, bool, str | None] + The structural shape, whether the field accepts `None`, and + the first `FieldInfo.description` encountered during unwrapping. + """ + return _unwrap( + annotation, + newtype_ctx=None, + owner=owner, + model_resolver=model_resolver, + union_resolver=union_resolver, + ) + - Tracks NewType names and refs during unwrapping: - - `outermost_newtype_name` / `outermost_newtype_ref`: the first - NewType encountered, exposed as `TypeInfo.newtype_name` / `newtype_ref`. - - `last_newtype_name`: the most recently entered NewType name, used - as the resolved `base_type` for the terminal type. - - `last_newtype_ref`: the most recently entered NewType callable, - used as constraint provenance (which NewType contributed each constraint). - - `newtype_outer_list_depth`: list layers accumulated before entering - the outermost NewType boundary. +def _unwrap( + annotation: object, + *, + newtype_ctx: _NewTypeCtx | None, + owner: type | None, + model_resolver: ModelResolver | None, + union_resolver: UnionResolver | None, +) -> tuple[FieldShape, bool, str | None]: + """Recurse one annotation layer, returning its `FieldShape` subtree. + + Parameters + ---------- + newtype_ctx + The innermost `NewType` currently in scope, or None. Sets the + terminal `Primitive.base_type` and tags constraints with their + contributing `NewType`. + owner + The model class supplying the namespace for forward-ref + resolution; invariant across the walk. + + Returns + ------- + tuple + The shape subtree, whether this layer or any descendant accepts + `None`, and the first `FieldInfo.description` found. """ - is_optional: bool = False - list_depth: int = 0 - newtype_outer_list_depth: int = 0 - is_dict: bool = False - dict_key_type: TypeInfo | None = None - dict_value_type: TypeInfo | None = None - constraints: list[ConstraintSource] = field(default_factory=list) - outermost_newtype_name: str | None = None - outermost_newtype_ref: object | None = None - last_newtype_name: str | None = None - last_newtype_ref: object | None = None - description: str | None = None - - def add_constraint(self, constraint: object) -> None: - self.constraints.append( - ConstraintSource(self.last_newtype_ref, self.last_newtype_name, constraint) + def _recurse( + annotation: object, newtype_ctx: _NewTypeCtx | None + ) -> tuple[FieldShape, bool, str | None]: + """Recurse into a child annotation, carrying the invariant resolvers.""" + return _unwrap( + annotation, + newtype_ctx=newtype_ctx, + owner=owner, + model_resolver=model_resolver, + union_resolver=union_resolver, ) - def build_type_info( - self, - *, - base_type: str, - kind: TypeKind, - literal_values: tuple[object, ...] | None = None, - source_type: type | None = None, - union_members: tuple[type[BaseModel], ...] | None = None, - ) -> TypeInfo: - return TypeInfo( - base_type=base_type, - kind=kind, - is_optional=self.is_optional, - list_depth=self.list_depth, - newtype_outer_list_depth=self.newtype_outer_list_depth, - is_dict=self.is_dict, - dict_key_type=self.dict_key_type, - dict_value_type=self.dict_value_type, - constraints=tuple(self.constraints), - literal_values=literal_values, - source_type=source_type, - newtype_name=self.outermost_newtype_name, - newtype_ref=self.outermost_newtype_ref, - union_members=union_members, - description=self.description, - ) + if isinstance(annotation, (str, ForwardRef)): + annotation = _resolve_forward_ref(annotation, owner) + + origin = get_origin(annotation) + if is_newtype(annotation): + ctx = _NewTypeCtx(annotation.__name__, annotation) # type: ignore[attr-defined] + inner, opt, desc = _recurse(annotation.__supertype__, ctx) # type: ignore[attr-defined] + inner = _erase_inner_newtypes(inner) + return NewTypeShape(name=ctx.name, ref=ctx.ref, inner=inner), opt, desc -def analyze_type(annotation: object) -> TypeInfo: - """Analyze a type annotation and return TypeInfo. + if origin is Annotated: + args = get_args(annotation) + inner_annotation = args[0] + own_desc: str | None = None + collected: list[ConstraintSource] = [] + for c in args[1:]: + if isinstance(c, FieldInfo): + if c.description is not None and own_desc is None: + own_desc = clean_docstring(c.description) + for m in c.metadata: + collected.append(_constraint_source(m, newtype_ctx)) + else: + collected.append(_constraint_source(c, newtype_ctx)) + + # Pick the annotation to recurse into and the optionality this + # Annotated layer contributes. A directly-wrapped union is peeled + # here so the resolver still sees the Annotated form; a `_Resolved` + # union short-circuits with the constraints attached. + next_annotation = inner_annotation + layer_optional = False + literal_alts: tuple[object, ...] = () + if _is_union(get_origin(inner_annotation)): + result = _peel_union( + inner_annotation, + union_resolver, + resolver_annotation=annotation, + description=own_desc, + ) + match result: + case _Resolved(shape): + return ( + attach_constraints(shape, tuple(collected)), + result.is_optional, + own_desc, + ) + case _ContinueWith(next_annotation, layer_optional, literal_alts): + pass + case _: + assert_never(result) + + if literal_alts: + collected.append(_literal_alternatives_source(literal_alts)) + inner, opt, desc = _recurse(next_annotation, newtype_ctx) + inner = attach_constraints(inner, tuple(collected)) + return ( + inner, + opt or layer_optional, + own_desc if own_desc is not None else desc, + ) - Iteratively unwraps type wrappers (Annotated, Optional, list, NewType) until - reaching a terminal type. + if _is_union(origin): + result = _peel_union(annotation, union_resolver) + match result: + case _Resolved(shape): + return shape, result.is_optional, None + case _ContinueWith(next_annotation, is_optional, literal_alts): + inner, opt, desc = _recurse(next_annotation, newtype_ctx) + if literal_alts: + inner = attach_constraints( + inner, (_literal_alternatives_source(literal_alts),) + ) + return inner, opt or is_optional, desc + case _: + assert_never(result) + + if origin is list: + args = get_args(annotation) + if not args: + raise TypeError("Bare list without type argument is not supported") + element, _, desc = _recurse(args[0], newtype_ctx) + # A list field is never optional on account of element nullability, + # so the element's `is_optional` is dropped; its description is the + # field's fallback prose when no field-level description exists. + return ArrayOf(element=element, constraints=()), False, desc + + if origin is dict: + args = get_args(annotation) + if not args: + raise TypeError("Bare dict without type arguments is not supported") + key_shape, _, _ = _recurse(args[0], None) + value_shape, _, _ = _recurse(args[1], None) + return MapOf(key=key_shape, value=value_shape, constraints=()), False, None + + return _terminal(annotation, newtype_ctx, model_resolver), False, None + + +def _resolve_forward_ref(annotation: str | ForwardRef, owner: type | None) -> object: + """Resolve a string / `ForwardRef` annotation to its type object. + + Resolves against *owner*'s module and class namespaces, plus *owner* + bound to its own name. The class namespace lets a forward ref to a + nested model (`Outer.Inner`) resolve; the self-name binding lets a + self-referential model defined in a local scope (e.g. a test body) + resolve `"Owner"` even when it is absent from the module globals. + Raises `UnresolvedForwardRefError` for a name not in scope + (`NameError`), a missing attribute on a dotted reference + (`AttributeError`), or a string that is not a valid type expression + (`SyntaxError`) -- a clean, named failure in place of the opaque + `TypeError` the terminal classifier would otherwise raise on a bare + string. """ - state = _UnwrapState() - - while True: - origin = get_origin(annotation) - - # Handle NewType (e.g., int32 = NewType("int32", Annotated[int, ...])) - if is_newtype(annotation): - name = annotation.__name__ # type: ignore[attr-defined] - state.last_newtype_name = name - state.last_newtype_ref = annotation - if state.outermost_newtype_name is None: - state.newtype_outer_list_depth = state.list_depth - state.outermost_newtype_name = name - state.outermost_newtype_ref = annotation - annotation = annotation.__supertype__ # type: ignore[attr-defined] - continue - - # Handle Annotated types (Annotated[X, metadata...]) - if origin is Annotated: - args = get_args(annotation) - annotation = args[0] - for c in args[1:]: - if isinstance(c, FieldInfo): - if c.description is not None and state.description is None: - state.description = clean_docstring(c.description) - for m in c.metadata: - state.add_constraint(m) - else: - state.add_constraint(c) - continue - - # Handle union types (X | None or Optional[X]) - if _is_union(origin): - args = get_args(annotation) - # Filter out None, Sentinel types (Pydantic's ), and - # Literal alternatives (e.g., HttpUrl | Literal[""] where the - # Literal is a special-value sentinel, not the primary type). - if any(a is types.NoneType for a in args): - state.is_optional = True - - non_none_args = [ - a - for a in args - if a is not types.NoneType and not isinstance(a, Sentinel) - ] - - # Only filter out Literal arms when a concrete (non-Literal) type - # exists. Without this guard, Optional[Literal["x"]] would lose - # all args because the Literal *is* the primary type. - concrete_args = [a for a in non_none_args if get_origin(a) is not Literal] - real_args = concrete_args if concrete_args else non_none_args - - if len(real_args) > 1: - # Check if all real args are BaseModel subclasses - # (unwrap Annotated wrappers to get the actual class) - members: list[type[BaseModel]] = [] - for arg in real_args: - inner = arg - if get_origin(inner) is Annotated: - inner = get_args(inner)[0] - if isinstance(inner, type) and issubclass(inner, BaseModel): - members.append(inner) - else: - raise UnsupportedUnionError( - f"Multi-type unions not supported: {annotation}" - ) - return state.build_type_info( - base_type=members[0].__name__, - kind=TypeKind.UNION, - union_members=tuple(members), - ) + if owner is not None: + localns = {**vars(owner), owner.__name__: owner} + else: + localns = None + try: + ref = ForwardRef(annotation) if isinstance(annotation, str) else annotation + return evaluate_forward_ref(ref, owner=owner, locals=localns) + except (NameError, SyntaxError, AttributeError) as exc: + target = ( + annotation if isinstance(annotation, str) else annotation.__forward_arg__ + ) + context = f" while extracting {owner.__qualname__}" if owner is not None else "" + raise UnresolvedForwardRefError( + f"Cannot resolve forward reference {target!r}{context}" + ) from exc + + +def _constraint_source( + constraint: object, newtype_ctx: _NewTypeCtx | None +) -> ConstraintSource: + return ConstraintSource( + source_ref=newtype_ctx.ref if newtype_ctx else None, + source_name=newtype_ctx.name if newtype_ctx else None, + constraint=constraint, + ) - if not real_args: - raise UnsupportedUnionError( - f"Union with no concrete types: {annotation}" - ) - annotation = real_args[0] - continue +def _literal_alternatives_source(values: tuple[object, ...]) -> ConstraintSource: + """Wrap dropped union `Literal` values as a `LiteralAlternatives` source.""" + return ConstraintSource( + source_ref=None, source_name=None, constraint=LiteralAlternatives(values) + ) - # Handle list types (list[X]) - if origin is list: - args = get_args(annotation) - if not args: - raise TypeError("Bare list without type argument is not supported") - state.list_depth += 1 - annotation = args[0] - continue - - # Handle dict types (dict[K, V]) - if origin is dict: - args = get_args(annotation) - if not args: - raise TypeError("Bare dict without type arguments is not supported") - state.is_dict = True - state.dict_key_type = analyze_type(args[0]) - state.dict_value_type = analyze_type(args[1]) - base_type = state.last_newtype_name or "dict" - return state.build_type_info( - base_type=base_type, - kind=TypeKind.PRIMITIVE, - source_type=dict, + +def _erase_inner_newtypes(shape: FieldShape) -> FieldShape: + """Drop every `NewTypeShape` reachable through `ArrayOf` layers. + + A `NewType` chain — including NewTypes nested as list elements — + collapses to a single `NewTypeShape` (the outermost), with inner + NewType names surviving only as the terminal `Primitive.base_type`. + Each `NewType` frame calls this on its recursion result so that by + the time the outermost frame returns, exactly one `NewTypeShape` + remains per spine. + + Recurses through `ArrayOf.element` but stops at `MapOf` — `dict` + key/value are independent spines, each keeping its own outermost + `NewTypeShape` — and at scalar / `ModelRef` / `UnionRef` terminals. + """ + match shape: + case NewTypeShape(inner=inner): + return _erase_inner_newtypes(inner) + case ArrayOf(element=element): + return replace(shape, element=_erase_inner_newtypes(element)) + case _: + return shape + + +def attach_constraints( + shape: FieldShape, constraints: tuple[ConstraintSource, ...] +) -> FieldShape: + """Prepend `constraints` to the outermost non-`NewTypeShape` layer. + + Skips any number of leading `NewTypeShape` wrappers, then prepends + to the `.constraints` of the first `ArrayOf`, `MapOf`, `Primitive`, + `LiteralScalar`, or `AnyScalar` reached. Does not descend into + `ArrayOf.element` or `MapOf.key` / `.value`. `ModelRef` / `UnionRef` + carry no constraints, so a constraint destined for a model/union + terminal (`Annotated[SomeModel, SomeConstraint()]`) raises + `NotImplementedError` rather than vanishing from both docs and + validation -- no current schema field does this, and silently + dropping it would diverge the generated output from the source. + + Length constraints (`annotated_types.MinLen` / `MaxLen`) are wrapped + into the typed `length_constraints` variants matching the + attachment layer: `ArrayMinLen` / `ArrayMaxLen` on `ArrayOf`, + `ScalarMinLen` / `ScalarMaxLen` on scalar layers. `MapOf` raises: + map-length constraints have no current schema use and would + otherwise silently take the scalar path. + """ + if not constraints: + return shape + match shape: + case NewTypeShape(inner=inner): + return replace(shape, inner=attach_constraints(inner, constraints)) + case ArrayOf(): + wrapped = tuple(_wrap_length_for_array(cs) for cs in constraints) + return replace(shape, constraints=wrapped + shape.constraints) + case MapOf(): + _reject_length_on_map(constraints) + return replace(shape, constraints=constraints + shape.constraints) + case Primitive() | LiteralScalar() | AnyScalar(): + wrapped = tuple(_wrap_length_for_scalar(cs) for cs in constraints) + return replace(shape, constraints=wrapped + shape.constraints) + case ModelRef() | UnionRef(): + names = ", ".join(type(cs.constraint).__name__ for cs in constraints) + raise NotImplementedError( + f"Constraints ({names}) on a model/union terminal are not " + f"supported; attach them to a scalar or array layer instead" ) + case _: + assert_never(shape) - break - return _classify_terminal(annotation, state) +def _wrap_length_for_array(cs: ConstraintSource) -> ConstraintSource: + """Replace a raw `MinLen`/`MaxLen` with its `ArrayOf`-layer variant. + Uses exact-type checks so already-wrapped variants (`ArrayMinLen`, + `ScalarMinLen`, etc.) are returned unchanged. + """ + if type(cs.constraint) is MinLen: + return replace(cs, constraint=ArrayMinLen(min_length=cs.constraint.min_length)) + if type(cs.constraint) is MaxLen: + return replace(cs, constraint=ArrayMaxLen(max_length=cs.constraint.max_length)) + return cs -def _classify_terminal(annotation: object, state: _UnwrapState) -> TypeInfo: - """Classify a fully-unwrapped terminal type into a TypeInfo.""" - # typing.Any -- treat as an opaque primitive - if annotation is Any: - return state.build_type_info( - base_type="Any", - kind=TypeKind.PRIMITIVE, - ) - # Literal types (e.g., Literal["value"] or Literal["a", "b"]) - if get_origin(annotation) is Literal: - args = get_args(annotation) - return state.build_type_info( - base_type="Literal", - kind=TypeKind.LITERAL, - literal_values=tuple(args), - ) +def _wrap_length_for_scalar(cs: ConstraintSource) -> ConstraintSource: + """Replace a raw `MinLen`/`MaxLen` with its scalar-layer variant. + Uses exact-type checks so already-wrapped variants (`ArrayMinLen`, + `ScalarMinLen`, etc.) are returned unchanged. + """ + if type(cs.constraint) is MinLen: + return replace(cs, constraint=ScalarMinLen(min_length=cs.constraint.min_length)) + if type(cs.constraint) is MaxLen: + return replace(cs, constraint=ScalarMaxLen(max_length=cs.constraint.max_length)) + return cs + + +def _reject_length_on_map(constraints: tuple[ConstraintSource, ...]) -> None: + """Raise on `MinLen`/`MaxLen` attached to a `MapOf` layer.""" + for cs in constraints: + if isinstance(cs.constraint, (MinLen, MaxLen)): + raise NotImplementedError( + f"{type(cs.constraint).__name__} on a Map type is not supported" + ) + + +def _terminal( + annotation: object, + newtype_ctx: _NewTypeCtx | None, + model_resolver: ModelResolver | None, +) -> FieldShape: + """Classify a fully-unwrapped terminal annotation into a shape.""" + if annotation is Any: + return AnyScalar(constraints=()) + if get_origin(annotation) is Literal: + return LiteralScalar(values=tuple(get_args(annotation)), constraints=()) if not isinstance(annotation, type): raise TypeError(f"Unsupported annotation type: {type(annotation)}") - if issubclass(annotation, list): raise TypeError("Bare list without type argument is not supported") - if issubclass(annotation, dict): raise TypeError("Bare dict without type arguments is not supported") + if issubclass(annotation, BaseModel) and model_resolver is not None: + return model_resolver(annotation) + base_type = newtype_ctx.name if newtype_ctx else annotation.__name__ + return Primitive(base_type=base_type, source_type=annotation, constraints=()) + + +def _peel_union( + annotation: object, + union_resolver: UnionResolver | None, + *, + resolver_annotation: object | None = None, + description: str | None = None, +) -> _ContinueWith | _Resolved: + """Process one union layer. + + Filters out `None` / `Sentinel` arms (recording `is_optional`), then + drops `Literal[...]` arms when a concrete (non-Literal) arm exists. + A single remaining arm is returned as `_ContinueWith`; multiple arms + invoke `union_resolver` and the result is returned as `_Resolved` + (raising `UnsupportedUnionError` when no resolver is supplied). + + `resolver_annotation` is passed to `union_resolver` instead of + `annotation` when set. This lets the `Annotated` branch forward the + full `Annotated[X | Y, ...]` form so resolvers can recover + discriminator metadata that the `Annotated` peeling step consumed. + """ + args = get_args(annotation) + is_optional = any(a is types.NoneType for a in args) + + non_none_args = _filter_sentinel_arms(args) + concrete_args = [a for a in non_none_args if get_origin(a) is not Literal] + real_args = concrete_args if concrete_args else non_none_args + + # A single concrete arm alongside `Literal[...]` arms keeps the concrete arm + # as the shape; the literal values ride along as a LiteralAlternatives + # constraint so they bypass the concrete arm's checks. Multi-arm and + # no-concrete-arm unions are unchanged (the literals stay dropped). + literal_alternatives: tuple[object, ...] = () + if len(concrete_args) == 1: + literal_args = [a for a in non_none_args if get_origin(a) is Literal] + literal_alternatives = tuple(v for a in literal_args for v in get_args(a)) + + if len(real_args) > 1: + members: list[type[BaseModel]] = [] + for arg in real_args: + inner = arg + if get_origin(inner) is Annotated: + inner = get_args(inner)[0] + if isinstance(inner, type) and issubclass(inner, BaseModel): + members.append(inner) + else: + raise UnsupportedUnionError( + f"Multi-type unions not supported: {annotation}" + ) + if union_resolver is None: + raise UnsupportedUnionError( + f"No union_resolver supplied for multi-arm union: {annotation}" + ) + return _Resolved( + union_resolver( + resolver_annotation or annotation, tuple(members), description + ), + is_optional, + ) - # Determine kind from type hierarchy - if issubclass(annotation, Enum): - kind = TypeKind.ENUM - elif issubclass(annotation, BaseModel): - kind = TypeKind.MODEL - else: - kind = TypeKind.PRIMITIVE + if not real_args: + raise UnsupportedUnionError(f"Union with no concrete types: {annotation}") - base_type = state.last_newtype_name or annotation.__name__ + return _ContinueWith(real_args[0], is_optional, literal_alternatives) - return state.build_type_info( - base_type=base_type, - kind=kind, - source_type=annotation, - ) + +def unwrap_list(annotation: object) -> object: + """Strip `| None`, `Sentinel`, and outermost `list[]` wrappers.""" + if _is_union(get_origin(annotation)): + args = _filter_sentinel_arms(get_args(annotation)) + if len(args) == 1: + annotation = args[0] + + while get_origin(annotation) is list: + annotation = get_args(annotation)[0] + return annotation def single_literal_value(annotation: object) -> object | None: - """Extract a single literal value from a type annotation, or None. + """Extract a single literal value from a type annotation, or `None`. - Delegates to analyze_type for all unwrapping, then checks - whether the result is a single-value Literal. Multi-value - Literals return None — callers needing all values should use - `analyze_type` and read `literal_values` directly. + Returns `None` for multi-value Literals -- callers needing all + values should use `analyze_type` and inspect the terminal + `LiteralScalar`'s `values`. """ try: - ti = analyze_type(annotation) + shape, _, _ = analyze_type(annotation) except (TypeError, UnsupportedUnionError): return None - if ( - ti.kind == TypeKind.LITERAL - and ti.literal_values - and len(ti.literal_values) == 1 - ): - return ti.literal_values[0] + terminal = terminal_of(shape) + if isinstance(terminal, LiteralScalar) and len(terminal.values) == 1: + return terminal.values[0] return None diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/type_registry.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/type_registry.py index 505657866..edb7a96fc 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/type_registry.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/type_registry.py @@ -1,113 +1,137 @@ """Type registry mapping Python types to target representations.""" from dataclasses import dataclass +from typing import Literal -from .type_analyzer import TypeInfo +from .field import FieldShape +from .field_walk import newtype_name, terminal_primitive __all__ = [ + "SparkCategory", "TypeMapping", "PRIMITIVE_TYPES", "get_type_mapping", "is_semantic_newtype", + "primitive_spark_category", "resolve_type_name", ] +SparkCategory = Literal["string", "int", "float", "bool", "other"] + @dataclass(frozen=True) class TypeMapping: """Maps a type to its representation in different targets.""" markdown: str - - def for_target(self, target: str) -> str: - """Get the type representation for a named target.""" - if target != "markdown": - raise ValueError(f"Unknown target {target!r}, expected 'markdown'") - return self.markdown + spark: str | None = None PRIMITIVE_TYPES: dict[str, TypeMapping] = { # Signed integers - "int8": TypeMapping(markdown="int8"), - "int16": TypeMapping(markdown="int16"), - "int32": TypeMapping(markdown="int32"), - "int64": TypeMapping(markdown="int64"), + "int8": TypeMapping(markdown="int8", spark="IntegerType()"), + "int16": TypeMapping(markdown="int16", spark="IntegerType()"), + "int32": TypeMapping(markdown="int32", spark="IntegerType()"), + "int64": TypeMapping(markdown="int64", spark="LongType()"), # Unsigned integers - "uint8": TypeMapping(markdown="uint8"), - "uint16": TypeMapping(markdown="uint16"), - "uint32": TypeMapping(markdown="uint32"), + "uint8": TypeMapping(markdown="uint8", spark="IntegerType()"), + "uint16": TypeMapping(markdown="uint16", spark="IntegerType()"), + "uint32": TypeMapping(markdown="uint32", spark="IntegerType()"), # Floating point - "float32": TypeMapping(markdown="float32"), - "float64": TypeMapping(markdown="float64"), + "float32": TypeMapping(markdown="float32", spark="FloatType()"), + "float64": TypeMapping(markdown="float64", spark="DoubleType()"), # Basic types - "str": TypeMapping(markdown="string"), - "bool": TypeMapping(markdown="boolean"), + "str": TypeMapping(markdown="string", spark="StringType()"), + "bool": TypeMapping(markdown="boolean", spark="BooleanType()"), # Python builtins (aliases to their portable equivalents) - "int": TypeMapping(markdown="int64"), - "float": TypeMapping(markdown="float64"), + "int": TypeMapping(markdown="int64", spark="LongType()"), + "float": TypeMapping(markdown="float64", spark="DoubleType()"), # Geometry types - "Geometry": TypeMapping(markdown="geometry"), + "Geometry": TypeMapping(markdown="geometry", spark="BinaryType()"), "BBox": TypeMapping(markdown="bbox"), } -def is_semantic_newtype(type_info: TypeInfo) -> bool: - """Whether a type represents a semantic NewType that should be displayed by name. +def is_semantic_newtype(shape: FieldShape) -> bool: + """Whether a shape's outermost NewType should be displayed by name. - Returns True for unregistered NewTypes (HexColor, Sources) and NewTypes - that wrap a different base type (FeatureVersion wrapping int32, Id wrapping - NoWhitespaceString). Returns False for registered primitives (int32, Geometry). + Returns True for unregistered NewTypes (HexColor, Sources) and + NewTypes that wrap a different base type (FeatureVersion wrapping + int32, Id wrapping NoWhitespaceString). Returns False for + registered primitives (int32, Geometry). """ - if type_info.newtype_name is None: + nt_name = newtype_name(shape) + if nt_name is None: return False - if type_info.newtype_name != type_info.base_type: + terminal = terminal_primitive(shape) + if terminal is None: return True - return get_type_mapping(type_info.base_type) is None + if nt_name != terminal.base_type: + return True + return get_type_mapping(terminal.base_type) is None def get_type_mapping(type_name: str) -> TypeMapping | None: """Look up a type mapping by name. - Parameters - ---------- - type_name : str - The type name to look up (e.g., "int32", "str", "Geometry"). - Also accepts Python builtin names ("int" -> int64, "float" -> float64). - - Returns - ------- - TypeMapping or None - The TypeMapping for the type, or None if not found. + Accepts portable type names (`int32`, `str`, `Geometry`) and Python + builtin names (`int` -> int64, `float` -> float64). """ return PRIMITIVE_TYPES.get(type_name) -def resolve_type_name(type_info: TypeInfo, target: str) -> str: - """Resolve a TypeInfo to the base type string for a given target. +# BinaryType() is intentionally absent: geometry maps to BinaryType() in +# PRIMITIVE_TYPES but falls through to "other" here, not a numeric/string/bool scalar. +_SPARK_TYPE_CATEGORIES: dict[str, SparkCategory] = { + "StringType()": "string", + "IntegerType()": "int", + "LongType()": "int", + "FloatType()": "float", + "DoubleType()": "float", + "BooleanType()": "bool", +} + - Looks up the type in the registry first (trying source_type if base_type - has no mapping). Falls back to the base_type name as-is. +def primitive_spark_category(base_type: str) -> SparkCategory: + """Return the Spark category for a primitive base type name. Parameters ---------- - type_info : TypeInfo - The analyzed type information. - target : str - The output target ("markdown"). + base_type + A primitive type name (`"int32"`, `"float64"`, `"bool"`, `"str"`, ...). Returns ------- - str - The resolved base type name string for the target. + SparkCategory + `"string"` for string-valued types, `"int"` for integer types, + `"float"` for floating-point types, `"bool"` for boolean types, + `"other"` for binary, geometry, or unregistered types. Unknown + types fall back to `"other"`, preserving string-default behavior + for any future unregistered type. + """ + mapping = get_type_mapping(base_type) + if mapping is None or mapping.spark is None: + return "other" + return _SPARK_TYPE_CATEGORIES.get(mapping.spark, "other") + + +def resolve_type_name(shape: FieldShape) -> str: + """Resolve a shape to its markdown base type name string. + + Looks up the terminal scalar's `base_type` in the registry first, + falling back to `source_type.__name__`. Semantic NewTypes wrapping + unregistered types resolve to the underlying class name (e.g. + `Sources` wrapping `SourceItem` -> `SourceItem`). """ - mapping = get_type_mapping(type_info.base_type) - if mapping is None and type_info.source_type is not None: - mapping = get_type_mapping(type_info.source_type.__name__) + terminal = terminal_primitive(shape) + if terminal is None: + return "?" + mapping = get_type_mapping(terminal.base_type) + if mapping is None and terminal.source_type is not None: + mapping = get_type_mapping(terminal.source_type.__name__) if mapping is not None: - return mapping.for_target(target) + return mapping.markdown - # Semantic NewType wrapping an unregistered type (e.g., Sources wrapping - # SourceItem): use the underlying class name rather than the NewType alias. - if type_info.newtype_name and type_info.source_type is not None: - return type_info.source_type.__name__ - return type_info.base_type + if newtype_name(shape) and terminal.source_type is not None: + return terminal.source_type.__name__ + return terminal.base_type diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/union_extraction.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/union_extraction.py index c555fdba0..45c8caca8 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/union_extraction.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/union_extraction.py @@ -2,6 +2,8 @@ from __future__ import annotations +from collections.abc import Mapping +from enum import Enum from typing import Annotated, get_args, get_origin from pydantic import BaseModel @@ -9,9 +11,24 @@ from overture.schema.system.feature import resolve_discriminator_field_name +from .field import ( + AnyScalar, + ArrayOf, + FieldShape, + LiteralScalar, + MapOf, + ModelRef, + NewTypeShape, + Primitive, + UnionRef, +) +from .field_walk import list_depth, terminal_of, walk_shape from .model_extraction import extract_model, resolve_field_alias -from .specs import AnnotatedField, UnionSpec, is_model_class -from .type_analyzer import TypeInfo, TypeKind, analyze_type, single_literal_value +from .specs import AnnotatedField, FieldSpec, MemberSpec, UnionSpec, is_model_class +from .type_analyzer import ( + capture_union_members, + single_literal_value, +) __all__ = ["extract_discriminator", "extract_union"] @@ -35,11 +52,11 @@ def _find_common_base(members: list[type[BaseModel]]) -> type[BaseModel]: def max_mro_index(cls: type) -> int: return max(mro.index(cls) for mro in filtered_mros) - return min(common, key=max_mro_index) + return min(common, key=lambda c: (max_mro_index(c), c.__module__, c.__qualname__)) def _find_field_by_alias(model: type[BaseModel], alias: str) -> FieldInfo | None: - """Find a field in model_fields by alias-resolved name.""" + """Find a field in `model_fields` by alias-resolved name.""" direct = model.model_fields.get(alias) if direct is not None: return direct @@ -73,18 +90,95 @@ def extract_discriminator( if field_info and field_info.annotation is not None: lit_val = single_literal_value(field_info.annotation) if lit_val is not None: - mapping[str(lit_val)] = member + key = lit_val.value if isinstance(lit_val, Enum) else str(lit_val) + mapping[key] = member return disc_field_name, mapping or None -_TypeShape = tuple[str, TypeKind, bool, int] +_TypeShape = tuple[object, ...] _FieldKey = tuple[str, _TypeShape] -def _type_shape(ti: TypeInfo) -> _TypeShape: - """Structural shape for dedup -- excludes source_type which varies across members.""" - return (ti.base_type, ti.kind, ti.is_optional, ti.list_depth) +def _structural_fingerprint(spec: FieldSpec) -> _TypeShape: + """Structural shape for dedup: ignores per-variant source_type variation. + + Two fields with the same name and same `(terminal_base_type, + terminal_kind, is_optional, list_depth)` collapse to a single + `AnnotatedField` whose `variant_sources` lists the contributing + members. + + `terminal_of` unwraps `ArrayOf` / `NewTypeShape`, so the terminal is + always one of the six leaf variants below; an unrecognized one + raises instead of silently collapsing into a shared fingerprint. + """ + depth = list_depth(spec.shape) + base_type: object + terminal = terminal_of(spec.shape) + match terminal: + case Primitive(base_type=bt): + base_type, kind = bt, "scalar" + case LiteralScalar(values=values): + base_type, kind = ("Literal", values), "scalar" + case AnyScalar(): + base_type, kind = "Any", "scalar" + case ModelRef(model=model): + base_type, kind = model.name, "model" + case UnionRef(union=union): + base_type, kind = union.name, "union" + case MapOf(): + base_type, kind = "dict", "map" + case _: + raise TypeError(f"Unexpected terminal shape: {terminal!r}") + return (base_type, kind, spec.is_optional, depth) + + +def _fingerprint_key(constraint: object) -> object: + """Return a value-stable set key for a single constraint. + + Constraints with value equality -- every `FieldConstraint`, the + `annotated_types` dataclasses, `GeometryTypeConstraint` -- key as + themselves. Foreign metadata that falls back to identity equality, namely + pydantic's internal `Field(...)` metadata, keys on its value-stable `repr` + so two equal-valued instances still collapse. + """ + if type(constraint).__eq__ is object.__eq__: + return repr(constraint) + return constraint + + +def _constraints_fingerprint(spec: FieldSpec) -> frozenset[object]: + """Constraints declared anywhere in *spec*'s shape tree, as a comparable set. + + `_structural_fingerprint` deliberately ignores constraints so that + members declaring the same field with per-variant `Annotated` + metadata still collapse to one `AnnotatedField`. This captures what + that ignores, so collisions with diverging constraints fail loudly + instead of silently keeping the last member's `FieldSpec`. + + Constraint identity lives on the constraints themselves: `FieldConstraint` + subclasses define value equality and hashing, so equal rules collapse in + the set. `_fingerprint_key` covers the lone foreign holdout that still + compares by identity. + """ + keys: list[object] = [] + + def collect(shape: FieldShape) -> None: + match shape: + case ( + Primitive(constraints=cs) + | LiteralScalar(constraints=cs) + | AnyScalar(constraints=cs) + | ArrayOf(constraints=cs) + | MapOf(constraints=cs) + ): + for source in cs: + keys.append(_fingerprint_key(source.constraint)) + case ModelRef() | UnionRef() | NewTypeShape(): + pass + + walk_shape(spec.shape, collect) + return frozenset(keys) def extract_union( @@ -92,39 +186,65 @@ def extract_union( annotation: object, *, entry_point: str | None = None, + partitions: Mapping[str, str] | None = None, ) -> UnionSpec: - """Extract a UnionSpec from a discriminated union type alias.""" - ti = analyze_type(annotation) - if ti.kind != TypeKind.UNION or ti.union_members is None: + """Extract a `UnionSpec` from a discriminated union type alias.""" + extracted = capture_union_members(annotation) + if extracted is None: raise TypeError(f"{name} is not a union type alias") + member_tuple, description = extracted + members = list(member_tuple) - members = list(ti.union_members) common_base = _find_common_base(members) + # Plain Python type aliases (`Foo = Annotated[...]`) don't preserve + # the alias name in the annotation. The nested-union path (called + # from extract_model for UNION-kind fields) passes `members[0].__name__` + # as the placeholder name. Recover the alias by convention: members + # extend `Base`, so stripping that suffix yields the alias. + # Top-level unions go through the CLI, which supplies the real name + # and skips this fallback. + # + # PEP 695 (`type Foo = Annotated[...]`) preserves `__name__` as + # `"Foo"` on 3.12+; after migrating, the placeholder hack can go. + member_names = {m.__name__ for m in members} + if name in member_names: + base_name = common_base.__name__ + name = ( + base_name.removesuffix("Base") if base_name.endswith("Base") else base_name + ) + base_spec = extract_model(common_base) shared_field_names = {f.name for f in base_spec.fields} - member_specs = [(m, extract_model(m)) for m in members] + member_specs = [MemberSpec(m, extract_model(m)) for m in members] annotated_fields: list[AnnotatedField] = [] - # Shared fields first (from common base) for fs in base_spec.fields: annotated_fields.append(AnnotatedField(field_spec=fs, variant_sources=None)) - # Variant-specific fields: collect by (name, type identity) for dedup seen: dict[_FieldKey, AnnotatedField] = {} - for member_cls, member_spec in member_specs: - for fs in member_spec.fields: + for member in member_specs: + member_cls = member.member_cls + for fs in member.spec.fields: if fs.name in shared_field_names: continue - key = (fs.name, _type_shape(fs.type_info)) + key = (fs.name, _structural_fingerprint(fs)) existing = seen.get(key) + if existing is not None: + existing_constraints = _constraints_fingerprint(existing.field_spec) + if _constraints_fingerprint(fs) != existing_constraints: + raise ValueError( + f"Union {name!r} field {fs.name!r} has the same structural " + f"shape across members but diverging constraints; dedup " + f"would silently drop one member's constraints" + ) prior_sources = existing.variant_sources or () if existing else () seen[key] = AnnotatedField( field_spec=fs, - variant_sources=(*prior_sources, member_cls.__name__), + variant_sources=(*prior_sources, member_cls), ) annotated_fields.extend(seen.values()) @@ -133,12 +253,14 @@ def extract_union( return UnionSpec( name=name, - description=ti.description, + description=description, annotated_fields=annotated_fields, members=members, + member_specs=member_specs, discriminator_field=disc_field, discriminator_mapping=disc_mapping, source_annotation=annotation, common_base=common_base, entry_point=entry_point, + partitions=partitions or {}, ) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/layout/module_layout.py b/packages/overture-schema-codegen/src/overture/schema/codegen/layout/module_layout.py index bb6b92379..f15bb0120 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/layout/module_layout.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/layout/module_layout.py @@ -10,6 +10,8 @@ from collections.abc import Iterable, Mapping from pathlib import PurePosixPath +from overture.schema.system.discovery import split_entry_point + __all__ = [ "OUTPUT_ROOT", "compute_output_dir", @@ -24,26 +26,13 @@ OUTPUT_ROOT = PurePosixPath(".") -def _split_entry_point(entry_point_path: str) -> tuple[str, str]: - """Split `"module.path:ClassName"` into its two parts. - - >>> _split_entry_point("overture.schema.buildings:Building") - ('overture.schema.buildings', 'Building') - """ - if ":" not in entry_point_path: - msg = f"Expected 'module:Class' format, got {entry_point_path!r}" - raise ValueError(msg) - module, cls = entry_point_path.split(":", 1) - return module, cls - - def entry_point_module(entry_point_path: str) -> str: """Extract module path from entry-point-style path. >>> entry_point_module("overture.schema.buildings:Building") 'overture.schema.buildings' """ - return _split_entry_point(entry_point_path)[0] + return split_entry_point(entry_point_path)[0] def entry_point_class(entry_point_path: str) -> str: @@ -52,7 +41,7 @@ def entry_point_class(entry_point_path: str) -> str: >>> entry_point_class("overture.schema.buildings:Building") 'Building' """ - return _split_entry_point(entry_point_path)[1] + return split_entry_point(entry_point_path)[1] def compute_schema_root(module_paths: Iterable[str]) -> str: diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/layout/type_collection.py b/packages/overture-schema-codegen/src/overture/schema/codegen/layout/type_collection.py index b9072da64..c4d597c13 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/layout/type_collection.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/layout/type_collection.py @@ -1,61 +1,64 @@ -"""Supplementary type discovery by walking expanded feature trees. +"""Supplementary type discovery by walking feature trees. -Walks FieldSpec.model references for sub-models (already extracted), -and extracts enums and NewTypes on first encounter. +Walks `FieldShape` trees to extract referenced enums, NewTypes, +Pydantic built-ins, and union member sub-models. `ModelRef` and +`UnionRef` carry their resolved specs structurally, so recursion +follows the shape directly. """ from collections.abc import Sequence +from enum import Enum from typing import Annotated, get_args, get_origin +from pydantic import BaseModel + from ..extraction.enum_extraction import extract_enum -from ..extraction.model_extraction import expand_model_tree, extract_model +from ..extraction.field import ( + FieldShape, + ModelRef, + NewTypeShape, + Primitive, + UnionRef, +) +from ..extraction.field_walk import walk_shape from ..extraction.newtype_extraction import extract_newtype from ..extraction.pydantic_extraction import extract_pydantic_type from ..extraction.specs import ( - FeatureSpec, FieldSpec, ModelSpec, + RecordSpec, SupplementarySpec, TypeIdentity, - is_pydantic_type, -) -from ..extraction.type_analyzer import ( - TypeInfo, - TypeKind, - analyze_type, - is_newtype, - walk_type_info, + is_pydantic_sourced, ) +from ..extraction.type_analyzer import analyze_type, is_newtype from ..extraction.type_registry import is_semantic_newtype __all__ = ["collect_all_supplementary_types"] def collect_all_supplementary_types( - feature_specs: Sequence[FeatureSpec], + model_specs: Sequence[ModelSpec], ) -> dict[TypeIdentity, SupplementarySpec]: """Collect supplementary types by walking expanded feature trees. - Requires that expand_model_tree has been called on all feature specs - first. Walks FieldSpec.model references for sub-models (already - extracted), and extracts enums and NewTypes on first encounter. - - Returns a dict mapping TypeIdentity to extracted specs. Two types - with the same class name from different modules are keyed separately. + Walks `ModelRef` references for sub-models (already extracted), + and extracts enums and NewTypes on first encounter. Two types + with the same class name from different modules are keyed + separately. """ - feature_objs: set[object] = {spec.identity.obj for spec in feature_specs} + feature_objs: set[object] = {spec.identity.obj for spec in model_specs} all_specs: dict[TypeIdentity, SupplementarySpec] = {} visited_models: set[object] = set() def _register_newtype(newtype_ref: object, name: str) -> bool: - """Register a NewType if not already present. Returns True if registered.""" nt_id = TypeIdentity(newtype_ref, name) if nt_id in all_specs: return False all_specs[nt_id] = extract_newtype(newtype_ref) return True - def _collect_from_model(model_spec: ModelSpec) -> None: + def _collect_from_model(model_spec: RecordSpec) -> None: if ( model_spec.source_type in visited_models or model_spec.source_type in feature_objs @@ -66,93 +69,54 @@ def _collect_from_model(model_spec: ModelSpec) -> None: _collect_from_fields(model_spec.fields) def _collect_inner_newtypes(newtype_ref: object) -> None: - """Walk a NewType's __supertype__ chain for intermediate semantic NewTypes.""" + """Walk a NewType's `__supertype__` chain for nested semantic NewTypes.""" annotation = getattr(newtype_ref, "__supertype__", None) while annotation is not None: if get_origin(annotation) is Annotated: annotation = get_args(annotation)[0] continue if is_newtype(annotation): - inner_ti = analyze_type(annotation) - if ( - inner_ti.newtype_ref is not None - and inner_ti.newtype_name is not None - and is_semantic_newtype(inner_ti) + inner_shape, _, _ = analyze_type(annotation) + if isinstance(inner_shape, NewTypeShape) and is_semantic_newtype( + inner_shape ): - _register_newtype(inner_ti.newtype_ref, inner_ti.newtype_name) + _register_newtype(inner_shape.ref, inner_shape.name) annotation = getattr(annotation, "__supertype__", None) continue break - def _collect_from_type_info(ti: TypeInfo) -> None: - """Collect supplementary types from a single TypeInfo. - - Uses walk_type_info for dict key/value recursion. Handles all - TypeKind variants without early returns so newtype extraction - and dict recursion apply regardless of kind. - """ - - def _visit(node: TypeInfo) -> None: - # UNION, ENUM, and pydantic (PRIMITIVE) are mutually exclusive - # by TypeKind. NewType extraction is orthogonal -- a node can be - # a NewType-wrapped ENUM, for instance. - if node.kind == TypeKind.UNION and node.union_members: - # Walk each member's fields for supplementary types. - # Members that are also top-level feature specs are skipped - # by the feature_objs guard in _collect_from_model. - for member_cls in node.union_members: - member_spec = extract_model(member_cls) - expand_model_tree(member_spec) - _collect_from_model(member_spec) - elif node.kind == TypeKind.ENUM and node.source_type is not None: - enum_id = TypeIdentity.of(node.source_type) - if enum_id not in all_specs: - all_specs[enum_id] = extract_enum(node.source_type) - elif is_pydantic_type(node): - if node.source_type is None: - raise TypeError( - "is_pydantic_type returned True but source_type is None" - ) - pid = TypeIdentity.of(node.source_type) - if pid not in all_specs: - all_specs[pid] = extract_pydantic_type(node.source_type) - - # Semantic NewTypes always get extracted, including intermediate - # NewTypes in the wrapping chain (e.g., Id wraps NoWhitespaceString - # wraps str -- both Id and NoWhitespaceString get pages). - if ( - node.newtype_ref is not None - and node.newtype_name is not None - and is_semantic_newtype(node) - ): - newly_registered = _register_newtype( - node.newtype_ref, node.newtype_name - ) - if newly_registered: - _collect_inner_newtypes(node.newtype_ref) - - walk_type_info(ti, _visit) + def _collect_from_shape(shape: FieldShape) -> None: + """Walk *shape* and register every supplementary type it touches.""" + + def _visit(node: FieldShape) -> None: + match node: + case NewTypeShape(name=name, ref=ref) if is_semantic_newtype(node): + if _register_newtype(ref, name): + _collect_inner_newtypes(ref) + case UnionRef(union=u): + for member in u.member_specs: + _collect_from_model(member.spec) + case ModelRef(model=m, starts_cycle=False): + _collect_from_model(m) + case Primitive(source_type=cls) if cls is not None and isinstance( + cls, type + ): + if issubclass(cls, Enum): + eid = TypeIdentity.of(cls) + if eid not in all_specs: + all_specs[eid] = extract_enum(cls) + elif is_pydantic_sourced(cls) and not issubclass(cls, BaseModel): + pid = TypeIdentity.of(cls) + if pid not in all_specs: + all_specs[pid] = extract_pydantic_type(cls) + + walk_shape(shape, _visit) def _collect_from_fields(fields: list[FieldSpec]) -> None: - # A single field can match multiple conditions (e.g., Sources is both - # a semantic NewType and wraps a MODEL-kind type), so checks are - # independent `if` statements, not `elif`. for field_spec in fields: - ti = field_spec.type_info - _collect_from_type_info(ti) - - # MODEL-kind fields (whether direct or via NewType wrapper) get expanded - if ti.kind == TypeKind.MODEL and ti.source_type is not None: - if field_spec.model is None: - msg = ( - f"MODEL-kind field {field_spec.name!r} has source_type " - f"but model=None — call expand_model_tree first" - ) - raise RuntimeError(msg) - if not field_spec.starts_cycle: - _collect_from_model(field_spec.model) - - for spec in feature_specs: + _collect_from_shape(field_spec.shape) + + for spec in model_specs: _collect_from_fields(spec.fields) return all_specs diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/link_computation.py b/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/link_computation.py index bf09950c4..a5c34fef7 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/link_computation.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/link_computation.py @@ -3,7 +3,8 @@ from dataclasses import dataclass from pathlib import PurePosixPath -from ..extraction.case_conversion import slug_filename +from overture.schema.system.case import to_snake_case + from ..extraction.specs import TypeIdentity __all__ = ["LinkContext", "relative_link"] @@ -28,7 +29,7 @@ def resolve_link_or_slug(self, identity: TypeIdentity) -> str: Always returns a usable link string. Use when the caller needs a link regardless of whether the type has a registered page. """ - return self.resolve_link(identity) or slug_filename(identity.name) + return self.resolve_link(identity) or f"{to_snake_case(identity.name)}.md" def _is_normalized(path: PurePosixPath) -> bool: diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/path_assignment.py b/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/path_assignment.py index f0d224ee4..7113ab8ca 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/path_assignment.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/path_assignment.py @@ -7,9 +7,10 @@ from collections.abc import Sequence from pathlib import PurePosixPath -from ..extraction.case_conversion import slug_filename +from overture.schema.system.case import to_snake_case + from ..extraction.specs import ( - FeatureSpec, + ModelSpec, PydanticTypeSpec, SupplementarySpec, TypeIdentity, @@ -29,7 +30,7 @@ def build_placement_registry( - feature_specs: Sequence[FeatureSpec], + model_specs: Sequence[ModelSpec], all_specs: dict[TypeIdentity, SupplementarySpec], numeric_names: list[TypeIdentity], geometry_names: list[TypeIdentity], @@ -45,7 +46,7 @@ def build_placement_registry( ) feature_dirs: set[PurePosixPath] = set() - for spec in feature_specs: + for spec in model_specs: spec_dir = output_dir_for_entry_point(spec.entry_point, schema_root) registry[spec.identity] = _md_path(spec_dir, spec.name) feature_dirs.add(spec_dir) @@ -54,10 +55,8 @@ def build_placement_registry( if tid in registry: continue if isinstance(supp_spec, PydanticTypeSpec): - registry[tid] = ( - PurePosixPath("pydantic") - / supp_spec.source_module - / slug_filename(tid.name) + registry[tid] = _md_path( + PurePosixPath("pydantic") / supp_spec.source_module, tid.name ) continue source_module = getattr(supp_spec.source_type, "__module__", None) @@ -77,7 +76,7 @@ def resolve_output_path( """Look up a type's output path from the registry, with flat-file fallback.""" if registry is not None and identity in registry: return registry[identity] - return PurePosixPath(slug_filename(identity.name)) + return _md_path(PurePosixPath(""), identity.name) def _aggregate_page_entries( @@ -112,4 +111,4 @@ def _nest_under_types( def _md_path(directory: PurePosixPath, name: str) -> PurePosixPath: """Build a .md file path from a directory and a PascalCase type name.""" - return directory / slug_filename(name) + return directory / f"{to_snake_case(name)}.md" diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/pipeline.py b/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/pipeline.py index f7c676c06..262782609 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/pipeline.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/pipeline.py @@ -14,14 +14,13 @@ from overture.schema.system.primitive import GeometryType from ..extraction.examples import ExampleRecord, load_examples -from ..extraction.model_extraction import expand_model_tree from ..extraction.numeric_extraction import extract_numerics from ..extraction.specs import ( EnumSpec, - FeatureSpec, ModelSpec, NewTypeSpec, PydanticTypeSpec, + RecordSpec, SupplementarySpec, TypeIdentity, UnionSpec, @@ -37,8 +36,8 @@ ) from .renderer import ( render_enum, - render_feature, render_geometry_from_values, + render_model, render_newtype, render_primitives_from_specs, render_pydantic_type, @@ -58,11 +57,11 @@ class RenderedPage: content: str path: PurePosixPath - is_feature: bool = False + is_model: bool = False def _load_model_examples( - spec: FeatureSpec, + spec: ModelSpec, ) -> list[ExampleRecord] | None: """Load examples for a feature spec, returning None when absent.""" if isinstance(spec, UnionSpec): @@ -97,16 +96,19 @@ def _render_supplement( ctx = LinkContext(output_path, registry) used_by = reverse_refs.get(tid) - if isinstance(spec, EnumSpec): - content = render_enum(spec, link_ctx=ctx, used_by=used_by) - elif isinstance(spec, NewTypeSpec): - content = render_newtype(spec, ctx, used_by=used_by) - elif isinstance(spec, ModelSpec): - content = render_feature(spec, ctx, used_by=used_by) - elif isinstance(spec, PydanticTypeSpec): - content = render_pydantic_type(spec, link_ctx=ctx, used_by=used_by) - else: - raise TypeError(f"Unhandled SupplementarySpec variant: {type(spec).__name__}") + match spec: + case EnumSpec(): + content = render_enum(spec, link_ctx=ctx, used_by=used_by) + case NewTypeSpec(): + content = render_newtype(spec, ctx, used_by=used_by) + case RecordSpec(): + content = render_model(spec, ctx, used_by=used_by) + case PydanticTypeSpec(): + content = render_pydantic_type(spec, link_ctx=ctx, used_by=used_by) + case _: + raise TypeError( + f"Unhandled SupplementarySpec variant: {type(spec).__name__}" + ) return RenderedPage(content=content, path=output_path) @@ -134,7 +136,7 @@ def partition_numeric_and_geometry_types( def generate_markdown_pages( - feature_specs: Sequence[FeatureSpec], + model_specs: Sequence[ModelSpec], schema_root: str, ) -> list[RenderedPage]: """Generate all markdown pages from feature specs. @@ -143,29 +145,25 @@ def generate_markdown_pages( I/O, frontmatter injection, and any output-format-specific concerns (like Docusaurus category files). """ - cache: dict[type, ModelSpec] = {} - for spec in feature_specs: - expand_model_tree(spec, cache) - numeric_names, geometry_names = partition_numeric_and_geometry_types( _system_primitive ) - all_specs = collect_all_supplementary_types(feature_specs) + all_specs = collect_all_supplementary_types(model_specs) registry = build_placement_registry( - feature_specs, all_specs, numeric_names, geometry_names, schema_root + model_specs, all_specs, numeric_names, geometry_names, schema_root ) - reverse_refs = compute_reverse_references(feature_specs, all_specs) + reverse_refs = compute_reverse_references(model_specs, all_specs) pages: list[RenderedPage] = [] - for spec in feature_specs: + for spec in model_specs: output_path = registry[spec.identity] ctx = LinkContext(output_path, registry) examples = _load_model_examples(spec) used_by = reverse_refs.get(spec.identity) - content = render_feature(spec, link_ctx=ctx, examples=examples, used_by=used_by) - pages.append(RenderedPage(content=content, path=output_path, is_feature=True)) + content = render_model(spec, link_ctx=ctx, examples=examples, used_by=used_by) + pages.append(RenderedPage(content=content, path=output_path, is_model=True)) for tid, supp_spec in all_specs.items(): pages.append(_render_supplement(tid, supp_spec, registry, reverse_refs)) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/renderer.py b/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/renderer.py index 0e829d1f4..10c96a803 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/renderer.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/renderer.py @@ -4,7 +4,7 @@ import functools import json import re -from collections.abc import Callable +from collections.abc import Callable, Iterable from dataclasses import dataclass from pathlib import Path from typing import TypedDict, cast @@ -14,23 +14,27 @@ from typing_extensions import NotRequired from ..extraction.examples import ExampleRecord +from ..extraction.field import ConstraintSource from ..extraction.field_constraints import constraint_display_text +from ..extraction.field_walk import ( + all_constraints, + list_depth, + map_key_value_constraints, + terminal_model_ref, +) from ..extraction.model_constraints import analyze_model_constraints from ..extraction.specs import ( AnnotatedField, EnumSpec, - FeatureSpec, FieldSpec, ModelSpec, NewTypeSpec, NumericSpec, PydanticTypeSpec, + RecordSpec, TypeIdentity, UnionSpec, ) -from ..extraction.type_analyzer import ( - ConstraintSource, -) from .link_computation import LinkContext from .reverse_references import UsedByEntry from .type_format import ( @@ -41,7 +45,7 @@ __all__ = [ "render_enum", - "render_feature", + "render_model", "render_geometry_from_values", "render_newtype", "render_primitives_from_specs", @@ -237,29 +241,34 @@ def _annotate_field_constraints( ) -> None: """Annotate a field row with constraints from the field's own annotation. - Shows constraints where source is None — those applied directly to + Shows constraints where source is None -- those applied directly to the field, not inherited from NewType chains. NewType-inherited constraints appear on the NewType's own page instead. """ link_fn = _link_fn_from_ctx(ctx) - notes = [ - constraint_display_text(cs, link_fn=link_fn) - for cs in field.type_info.constraints - if cs.source_ref is None - ] + + def directly_applied(prefix: str, sources: Iterable[ConstraintSource]) -> list[str]: + return [ + f"{prefix}{constraint_display_text(cs, link_fn=link_fn)}" + for cs in sources + if cs.source_ref is None + ] + + key_constraints, value_constraints = map_key_value_constraints(field.shape) + notes = directly_applied("", all_constraints(field.shape)) + notes += directly_applied("key: ", key_constraints) + notes += directly_applied("value: ", value_constraints) if notes: _annotate_constraint_notes(row, notes) def _expandable_list_suffix(field_spec: FieldSpec) -> str: """Return `"[]"` per nesting level for list-of-model fields expanded inline.""" - if ( - field_spec.type_info.is_list - and field_spec.model - and not field_spec.starts_cycle - ): - return "[]" * field_spec.type_info.list_depth - return "" + model_ref = terminal_model_ref(field_spec.shape) + if model_ref is None or model_ref.starts_cycle: + return "" + depth = list_depth(field_spec.shape) + return "[]" * depth if depth > 0 else "" def _expand_sub_model( @@ -269,10 +278,13 @@ def _expand_sub_model( result: list[_FieldRow], ) -> None: """Expand sub-model fields inline, appending child rows to *result*.""" - sub = field_spec.model if not field_spec.starts_cycle else None - if sub is not None: - child_prefix = f"{name}{_expandable_list_suffix(field_spec)}." - result.extend(_expand_model_fields(sub.fields, ctx, prefix=child_prefix)) + model_ref = terminal_model_ref(field_spec.shape) + if model_ref is None or model_ref.starts_cycle: + return + child_prefix = f"{name}{_expandable_list_suffix(field_spec)}." + result.extend( + _expand_model_fields(model_ref.model.fields, ctx, prefix=child_prefix) + ) def _annotate_top_level_constraints( @@ -341,7 +353,7 @@ def _variant_tag(annotated: AnnotatedField, union_name: str) -> str | None: if annotated.variant_sources is None: return None short_names = [ - _short_variant_name(v, union_name) for v in annotated.variant_sources + _short_variant_name(v.__name__, union_name) for v in annotated.variant_sources ] return f" *({', '.join(short_names)})*" @@ -379,15 +391,14 @@ def _expand_union_fields( return result -def render_feature( - spec: FeatureSpec, +def render_model( + spec: ModelSpec, link_ctx: LinkContext | None = None, examples: list[ExampleRecord] | None = None, used_by: list[UsedByEntry] | None = None, ) -> str: - """Render a FeatureSpec (ModelSpec or UnionSpec) as Markdown documentation. + """Render a feature spec as Markdown documentation. - For ModelSpec, requires expand_model_tree to have been called first. For UnionSpec, adds inline variant tags to variant-specific fields. """ template = _get_jinja_env().get_template("feature.md.jinja2") @@ -396,7 +407,7 @@ def render_feature( if isinstance(spec, UnionSpec): fields = _expand_union_fields(spec, link_ctx, constraint_notes=field_notes) - elif isinstance(spec, ModelSpec): + elif isinstance(spec, RecordSpec): fields = _expand_model_fields(spec.fields, link_ctx) _annotate_top_level_constraints(fields, field_notes) else: @@ -491,13 +502,13 @@ def render_newtype( link_ctx: LinkContext | None = None, used_by: list[UsedByEntry] | None = None, ) -> str: - """Render a NewTypeSpec as Markdown documentation.""" + """Render a `NewTypeSpec` as Markdown documentation.""" template = _get_jinja_env().get_template("newtype.md.jinja2") - ti = newtype_spec.type_info - underlying = format_underlying_type(ti, link_ctx) + shape = newtype_spec.shape + underlying = format_underlying_type(shape, link_ctx) constraints = [ _format_constraint(cs, newtype_spec.source_type, link_ctx) - for cs in ti.constraints + for cs in all_constraints(shape) ] return template.render( diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/reverse_references.py b/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/reverse_references.py index 2ad471fc1..91f2864bc 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/reverse_references.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/reverse_references.py @@ -6,17 +6,26 @@ from dataclasses import dataclass from enum import Enum +from pydantic import BaseModel + +from ..extraction.field import ( + FieldShape, + ModelRef, + NewTypeShape, + Primitive, + UnionRef, +) +from ..extraction.field_walk import all_constraints, walk_shape from ..extraction.specs import ( - FeatureSpec, FieldSpec, ModelSpec, NewTypeSpec, + RecordSpec, SupplementarySpec, TypeIdentity, UnionSpec, - is_pydantic_type, + is_pydantic_sourced, ) -from ..extraction.type_analyzer import TypeInfo, TypeKind, walk_type_info __all__ = [ "UsedByEntry", @@ -41,7 +50,7 @@ class UsedByEntry: def compute_reverse_references( - feature_specs: Sequence[FeatureSpec], + model_specs: Sequence[ModelSpec], all_specs: Mapping[TypeIdentity, SupplementarySpec], ) -> dict[TypeIdentity, list[UsedByEntry]]: """Compute reverse references from types to their referrers. @@ -51,121 +60,108 @@ def compute_reverse_references( Parameters ---------- - feature_specs : Sequence[FeatureSpec] - Feature-level specs (ModelSpec or UnionSpec). - all_specs : Mapping[TypeIdentity, SupplementarySpec] + model_specs + Feature-level specs (RecordSpec or UnionSpec). + all_specs Supplementary types (enums, newtypes, sub-models). - - Returns - ------- - dict[TypeIdentity, list[UsedByEntry]] - Dict mapping TypeIdentity to sorted lists of UsedByEntry. """ - # Track references with sets to deduplicate - references: dict[TypeIdentity, set[UsedByEntry]] = {} + # An insertion-ordered set (dict keys) per target: dedups like a set but + # iterates deterministically, so sorted()'s stable order breaks ties by + # insertion rather than by nondeterministic set-hash order. + references: dict[TypeIdentity, dict[UsedByEntry, None]] = {} def add_reference( target: TypeIdentity, referrer: TypeIdentity, kind: UsedByKind ) -> None: - """Add a reference from referrer to target, with deduplication.""" if target == referrer or target not in all_specs: return - references.setdefault(target, set()).add(UsedByEntry(referrer, kind)) + references.setdefault(target, {})[UsedByEntry(referrer, kind)] = None - def collect_from_type_info( - ti: TypeInfo, referrer: TypeIdentity, referrer_kind: UsedByKind + def collect_from_shape( + shape: FieldShape, + referrer: TypeIdentity, + referrer_kind: UsedByKind, ) -> None: - """Collect references from a TypeInfo.""" - - def _visit(node: TypeInfo) -> None: - if node.newtype_ref is not None and node.newtype_name is not None: - add_reference( - TypeIdentity(node.newtype_ref, node.newtype_name), - referrer, - referrer_kind, - ) + """Walk a shape and add references for every type it touches.""" - # ENUM, MODEL, pydantic (PRIMITIVE), and UNION are mutually - # exclusive by TypeKind. - if ( - node.kind in (TypeKind.ENUM, TypeKind.MODEL) - and node.source_type is not None - ): - add_reference( - TypeIdentity.of(node.source_type), - referrer, - referrer_kind, - ) - elif is_pydantic_type(node): - add_reference( - TypeIdentity.of(node.source_type), referrer, referrer_kind - ) - elif node.union_members is not None: - for member_cls in node.union_members: + def _visit(node: FieldShape) -> None: + match node: + case NewTypeShape(name=name, ref=ref): + add_reference(TypeIdentity(ref, name), referrer, referrer_kind) + case ModelRef(model=m) if m.source_type is not None: add_reference( - TypeIdentity.of(member_cls), - referrer, - referrer_kind, + TypeIdentity.of(m.source_type), referrer, referrer_kind ) - - walk_type_info(ti, _visit) + case UnionRef(union=u): + for member_cls in u.members: + add_reference( + TypeIdentity.of(member_cls), referrer, referrer_kind + ) + case Primitive(source_type=cls) if cls is not None: + if isinstance(cls, type) and ( + issubclass(cls, Enum) + or issubclass(cls, BaseModel) + or is_pydantic_sourced(cls) + ): + add_reference(TypeIdentity.of(cls), referrer, referrer_kind) + + walk_shape(shape, _visit) def collect_from_fields( - fields: list[FieldSpec], referrer: TypeIdentity, referrer_kind: UsedByKind + fields: list[FieldSpec], + referrer: TypeIdentity, + referrer_kind: UsedByKind, ) -> None: - """Collect references from model fields.""" + """Collect references from each field's shape.""" for field_spec in fields: - collect_from_type_info(field_spec.type_info, referrer, referrer_kind) + collect_from_shape(field_spec.shape, referrer, referrer_kind) - def collect_from_model_spec(spec: ModelSpec, referrer: TypeIdentity) -> None: - """Collect references from a ModelSpec.""" + def collect_from_model_spec(spec: RecordSpec, referrer: TypeIdentity) -> None: collect_from_fields(spec.fields, referrer, UsedByKind.MODEL) def collect_from_union_spec(spec: UnionSpec) -> None: - """Collect references from a UnionSpec.""" referrer = spec.identity # Union features reference their members for member_cls in spec.members: - add_reference( - TypeIdentity.of(member_cls), - referrer, - UsedByKind.MODEL, - ) - # Also walk fields for other supplementary types + add_reference(TypeIdentity.of(member_cls), referrer, UsedByKind.MODEL) collect_from_fields(spec.fields, referrer, UsedByKind.MODEL) def collect_from_newtype_spec(spec: NewTypeSpec, referrer: TypeIdentity) -> None: - """Collect references from a NewTypeSpec.""" - collect_from_type_info(spec.type_info, referrer, UsedByKind.NEWTYPE) - - # Collect inherited NewTypes from constraint sources - for cs in spec.type_info.constraints: + # The NewType's own identity isn't added here (self-reference). + # spec.shape already has the outer NewTypeShape stripped. + collect_from_shape(spec.shape, referrer, UsedByKind.NEWTYPE) + + # Inherited NewTypes from constraint sources at every layer + # (array / map / scalar), not just the terminal scalar -- a + # NewType chaining through an array NewType carries the inner + # NewType's provenance on the array layer. + for cs in all_constraints(spec.shape): if cs.source_ref is not None and cs.source_name is not None: - ref_id = TypeIdentity(cs.source_ref, cs.source_name) - add_reference(ref_id, referrer, UsedByKind.NEWTYPE) + add_reference( + TypeIdentity(cs.source_ref, cs.source_name), + referrer, + UsedByKind.NEWTYPE, + ) # Collect from features - for spec in feature_specs: - if isinstance(spec, ModelSpec): + for spec in model_specs: + if isinstance(spec, RecordSpec): collect_from_model_spec(spec, spec.identity) elif isinstance(spec, UnionSpec): collect_from_union_spec(spec) - # Collect from supplementary specs (NewTypes and sub-models reference - # other types; enums do not, so they need no processing here) + # Collect from supplementary specs (enums have no outgoing references) for tid, supp_spec in all_specs.items(): if isinstance(supp_spec, NewTypeSpec): collect_from_newtype_spec(supp_spec, tid) - elif isinstance(supp_spec, ModelSpec): + elif isinstance(supp_spec, RecordSpec): collect_from_model_spec(supp_spec, tid) - # Sort into deterministic lists. (kind, name) handles the common case; - # module breaks ties when two referrers share the same display name - # (e.g. identically-named types from different themes/modules). + # Sort into deterministic lists. result: dict[TypeIdentity, list[UsedByEntry]] = {} - for target, ref_set in references.items(): + for target, ref_map in references.items(): entries = sorted( - ref_set, + ref_map, key=lambda e: (e.kind.value, e.identity.name, e.identity.module), ) result[target] = entries diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/type_format.py b/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/type_format.py index b6bd7a6ec..55dde02cc 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/type_format.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/markdown/type_format.py @@ -1,16 +1,34 @@ -"""Format TypeInfo as markdown type strings with cross-page links.""" +"""Format `FieldShape` trees as markdown type strings with cross-page links.""" from __future__ import annotations -from pydantic import BaseModel +from collections.abc import Sequence +from enum import Enum -from ..extraction.specs import FieldSpec, TypeIdentity -from ..extraction.type_analyzer import TypeInfo, TypeKind -from ..extraction.type_registry import is_semantic_newtype, resolve_type_name +from pydantic import BaseModel +from typing_extensions import assert_never + +from ..extraction.field import ( + AnyScalar, + ArrayOf, + FieldShape, + LiteralScalar, + MapOf, + ModelRef, + NewTypeShape, + Primitive, + Scalar, + UnionRef, +) +from ..extraction.specs import FieldSpec, TypeIdentity, is_pydantic_sourced +from ..extraction.type_registry import ( + get_type_mapping, + is_semantic_newtype, + resolve_type_name, +) from .link_computation import LinkContext __all__ = [ - "format_dict_type", "format_type", "format_underlying_type", "resolve_type_link", @@ -18,17 +36,17 @@ def _code_link(name: str, href: str) -> str: - """Format a markdown link with inline-code text: [`name`](href).""" + """Format a markdown link with inline-code text: `[``name``](href)`.""" return f"[`{name}`]({href})" def resolve_type_link(identity: TypeIdentity, ctx: LinkContext | None = None) -> str: - """Resolve a TypeIdentity to a linked code span or plain code span. + """Resolve a `TypeIdentity` to a linked or plain code span. - When *ctx* is provided, links only to types in the registry (types - without pages render as inline code). Without context, renders as - inline code -- producing a link requires a placement registry to - compute correct relative paths. + With `ctx`, links only to types in the registry (types without + pages render as inline code). Without context, renders as inline + code -- producing a link requires a placement registry to compute + correct relative paths. """ if ctx: href = ctx.resolve_link(identity) @@ -40,9 +58,9 @@ def resolve_type_link(identity: TypeIdentity, ctx: LinkContext | None = None) -> def _wrap_list_n(inner: str, depth: int) -> str: """Wrap an inner type string in `list<...>` markdown syntax *depth* times. - Builds a single broken-backtick wrapper rather than nesting iteratively. - Iterative nesting creates adjacent backticks that CommonMark - interprets as multi-backtick code span delimiters. + Builds a single broken-backtick wrapper rather than nesting + iteratively, since iterative nesting creates adjacent backticks + that CommonMark interprets as multi-backtick code span delimiters. """ return f"`{'list<' * depth}`{inner}`{'>' * depth}`" @@ -52,189 +70,314 @@ def _plain_list_type(base: str, depth: int) -> str: return f"`{'list<' * depth}{base}{'>' * depth}`" -def _linked_type_identity(ti: TypeInfo) -> TypeIdentity | None: - """Return the TypeIdentity to use for a markdown link, or None for non-linked types.""" - if ( - is_semantic_newtype(ti) - and ti.newtype_ref is not None - and ti.newtype_name is not None - ): - return TypeIdentity(ti.newtype_ref, ti.newtype_name) - if ti.kind in (TypeKind.ENUM, TypeKind.MODEL) and ti.source_type is not None: - return TypeIdentity(ti.source_type, ti.base_type) - return None +def _peel_arrays(shape: FieldShape) -> tuple[int, FieldShape]: + """Strip outer `ArrayOf` layers; return (count, inner).""" + depth = 0 + while isinstance(shape, ArrayOf): + depth += 1 + shape = shape.element + return depth, shape + +def _format_literal(values: tuple[object, ...]) -> str: + """Format Literal values for display.""" + if len(values) == 1: + return f'`"{values[0]}"`' + return r" \| ".join(f'`"{v}"`' for v in values) -def _try_primitive_link( - ti: TypeInfo, display_name: str, ctx: LinkContext | None -) -> str | None: - """Try to link a PRIMITIVE type to its page via registry lookup. - Registered primitives (int32, Geometry) and Pydantic types (HttpUrl) - can have pages in the registry. Uses the type registry display name - (e.g. `geometry` not `Geometry`) for the link text. +def _format_union_members( + members: Sequence[type[BaseModel]], + ctx: LinkContext | None, + separator: str = r" \| ", +) -> str: + r"""Format union members as individually linked / backticked names. + + Each member is resolved independently so members with pages get + linked while others render as plain code spans. `separator` is + inserted between members (default is `\|` for table-cell safety). """ - if ti.kind != TypeKind.PRIMITIVE or not ctx: + return separator.join(resolve_type_link(TypeIdentity.of(m), ctx) for m in members) + + +def _model_ref_identity(model_ref: ModelRef) -> TypeIdentity | None: + """Return a linkable identity for a `ModelRef`, or None when unsourced. + + A `ModelRef` links by its `source_type` (the original Python class) + paired with the model name. Returns None when `source_type` is absent + -- a synthesized spec with no backing class has no page to link to. + """ + src = model_ref.model.source_type + if src is None: return None - candidate = ti.newtype_ref or ti.source_type - if candidate is None: + return TypeIdentity(src, model_ref.model.name) + + +def _model_link(model_ref: ModelRef, ctx: LinkContext | None) -> str: + """Resolve a `ModelRef` to a markdown link or fallback code span.""" + identity = _model_ref_identity(model_ref) + if identity is not None: + return resolve_type_link(identity, ctx) + return f"`{model_ref.model.name}`" + + +def _scalar_identity(scalar: Primitive) -> TypeIdentity | None: + """Return a linkable identity for a `Primitive`'s `source_type`, if any. + + Enum / BaseModel / Pydantic-sourced types link by their own + identity and class name. Class-based registered primitives + (`Geometry`, `BBox`) are plain classes -- not BaseModel, not + Pydantic-sourced -- so they link by object identity to their + aggregate page under the markdown registry name (`geometry`, `bbox`). + """ + src = scalar.source_type + if not isinstance(src, type): return None - href = ctx.resolve_link(TypeIdentity(candidate, display_name)) - if href: - return _code_link(display_name, href) + if issubclass(src, Enum) or issubclass(src, BaseModel) or is_pydantic_sourced(src): + return TypeIdentity.of(src) + if get_type_mapping(src.__name__) is not None: + return TypeIdentity(src, _registry_name(scalar)) return None -def _markdown_type_name(ti: TypeInfo) -> str: - """Return the markdown display name for a type. +def _scalar_display(scalar: Scalar, ctx: LinkContext | None) -> tuple[str, bool]: + """Render a `Scalar` variant as a markdown string; second value is True if linked. - Uses the semantic NewType name when present (e.g. `LanguageTag`), - otherwise falls back to the resolved markdown type (e.g. `string`). + Linked when the scalar is a `Primitive` with an Enum / BaseModel / + Pydantic-sourced `source_type` whose identity resolves to a page. + Otherwise renders as the registry-resolved markdown name. + """ + if isinstance(scalar, Primitive): + identity = _scalar_identity(scalar) + if identity is not None and ctx: + href = ctx.resolve_link(identity) + if href: + return _code_link(identity.name, href), True + if identity is not None: + return f"`{identity.name}`", False + return f"`{_registry_name(scalar)}`", False + + +def _registry_name(scalar: Scalar) -> str: + """Resolve a scalar to its markdown registry name (e.g. `int64`).""" + if isinstance(scalar, LiteralScalar): + return "Literal" + if isinstance(scalar, AnyScalar): + return "Any" + mapping = get_type_mapping(scalar.base_type) + if mapping is None and scalar.source_type is not None: + mapping = get_type_mapping(scalar.source_type.__name__) + if mapping is not None: + return mapping.markdown + return scalar.base_type + + +def _format_map(shape: MapOf, ctx: LinkContext | None) -> str: + """Format a `MapOf` as a `map` code span, linking key/value types. + + Semantic NewTypes and Enum / BaseModel-sourced key/value types link + to their pages; primitives stay bare. Output is identical whether the + map is rendered in a field cell or as a NewType's underlying type -- + both paths route through here. + + A link has to break out of the surrounding code span, so any bare side + is folded into the adjacent `map<...>` span rather than wrapped in its + own backticks. Two backtick spans must never abut: CommonMark reads the + resulting `` as a two-backtick delimiter and swallows the link. """ - name = ti.newtype_name if is_semantic_newtype(ti) else None - return name or resolve_type_name(ti, "markdown") + key_str, key_linked = _map_side(shape.key, ctx) + val_str, val_linked = _map_side(shape.value, ctx) + if not key_linked and not val_linked: + return f"`map<{key_str}, {val_str}>`" + if key_linked and val_linked: + return f"`map<`{key_str}`,`{val_str}`>`" + if key_linked: + return f"`map<`{key_str}`,{val_str}>`" + return f"`map<{key_str},`{val_str}`>`" -def format_dict_type(ti: TypeInfo) -> str: - """Format a dict TypeInfo as bare `map` using resolved markdown names.""" - if ti.dict_key_type is None or ti.dict_value_type is None: - msg = f"format_dict_type requires dict key/value types, got {ti}" - raise ValueError(msg) - key = _markdown_type_name(ti.dict_key_type) - value = _markdown_type_name(ti.dict_value_type) - return f"map<{key}, {value}>" +def _map_side(shape: FieldShape, ctx: LinkContext | None) -> tuple[str, bool]: + """Render one map key/value as (text, is_link). + Returns a page link when the side resolves to one, else its + container-aware bare name (so a `list<...>` / `map<...>` wrapper + survives instead of collapsing to its element). The flag tells + `_format_map` whether the side breaks out of the surrounding code span. + """ + link = _map_side_link(shape, ctx) + if link is not None: + return link, True + return _bare_map_side_name(shape), False + + +def _map_side_link(shape: FieldShape, ctx: LinkContext | None) -> str | None: + """Return a markdown link for a map key/value that has its own page. + + Links a semantic NewType, a model (`ModelRef`), or a primitive whose + `source_type` is a linkable identity (`_scalar_identity`), when `ctx` + resolves a page for it. NewType and primitive sides link through + `list<...>` layers; a model side links only when it is the direct map + side (`depth == 0`), so a `list`-valued map keeps its `list<...>` + wrapper from `_bare_map_side_name` rather than collapsing to a bare model + link. Returns None when the side has no page; the caller renders a bare + name instead. + """ + identity: TypeIdentity | None = None + depth, cur = _peel_arrays(shape) + if isinstance(cur, NewTypeShape) and is_semantic_newtype(shape): + identity = TypeIdentity(cur.ref, cur.name) + elif depth == 0 and isinstance(cur, ModelRef): + identity = _model_ref_identity(cur) + elif isinstance(cur, Primitive): + identity = _scalar_identity(cur) + if identity and ctx: + href = ctx.resolve_link(identity) + if href: + return _code_link(identity.name, href) + return None -def _format_union_members( - members: tuple[type[BaseModel], ...], - ctx: LinkContext | None, - separator: str = r" \| ", -) -> str: - r"""Format union members as individually linked/backticked names. - Each member is resolved independently so members with pages get linked - while others render as plain code spans. *separator* is inserted between - members (default is `\|` for table-cell safety). - """ - return separator.join(resolve_type_link(TypeIdentity.of(m), ctx) for m in members) +def _bare_map_side_name(shape: FieldShape) -> str: + """Bare markdown name for a map key/value, recursing through containers. + Every variant resolves to a real name: `list<...>` / `map<...>` + wrappers recurse, scalars use their registry name (so `Any` is `Any`, + not `?`), semantic NewTypes and models use their type name, and a + pass-through NewType resolves through the registry like the scalar it + aliases. There is no `?` fallback -- a side that can't be named is a + bug, not a placeholder. -def format_type( - field: FieldSpec, - ctx: LinkContext | None = None, -) -> str: + A union-valued map is the one shape left unrendered: no schema field + uses one, and its `\\|`-separated members do not compose cleanly into + a bare `map<...>` span. It raises so the gap surfaces loudly when a + field first needs it, rather than shipping a half-rendered value. + """ + match shape: + case ArrayOf(element=element): + return f"list<{_bare_map_side_name(element)}>" + case MapOf(key=key, value=value): + return f"map<{_bare_map_side_name(key)}, {_bare_map_side_name(value)}>" + case NewTypeShape(name=name) if is_semantic_newtype(shape): + return name + case NewTypeShape(): + return resolve_type_name(shape) + case ModelRef(model=model): + return model.name + case Primitive() | LiteralScalar() | AnyScalar(): + return _registry_name(shape) + case UnionRef(): + raise NotImplementedError( + "union-typed map key/value is not rendered in markdown; " + "add handling here when a schema field first needs one" + ) + case _: + assert_never(shape) + + +def format_type(field: FieldSpec, ctx: LinkContext | None = None) -> str: """Format a field's type for markdown display, with links and qualifiers.""" - ti = field.type_info qualifiers: list[str] = [] - - if ti.kind == TypeKind.LITERAL and ti.literal_values: - if len(ti.literal_values) == 1: - return f'`"{ti.literal_values[0]}"`' - return r" \| ".join(f'`"{v}"`' for v in ti.literal_values) - - identity = _linked_type_identity(ti) - - if ti.kind == TypeKind.UNION and ti.union_members: - display = _format_union_members(ti.union_members, ctx) - if ti.is_list: - qualifiers.append("list") - elif ti.is_dict: - if identity: - display = resolve_type_link(identity, ctx) - qualifiers.append("map") - else: - display = f"`{format_dict_type(ti)}`" - elif identity: - display = resolve_type_link(identity, ctx) - # List layers outside a NewType wrap with list<> syntax (e.g., list[PhoneNumber] - # renders as list). List layers inside a NewType use a (list) - # qualifier instead (e.g., Sources wrapping list[SourceItem] renders as - # Sources (list)), since the list-ness is an implementation detail of the type. - if ti.newtype_outer_list_depth > 0: - display = _wrap_list_n(display, ti.newtype_outer_list_depth) - elif ti.is_list and ti.newtype_name is not None: # list is inside the NewType - qualifiers.append("list") - elif ti.is_list: - display = _wrap_list_n(display, ti.list_depth) - else: - # Fallback: types without a linked identity. Registered primitives (int32, - # Geometry) and Pydantic types (HttpUrl) may still link to aggregate pages - # via the placement registry. Unregistered primitives render as plain code. - base = resolve_type_name(ti, "markdown") - link = _try_primitive_link(ti, base, ctx) - if link and ti.is_list: - display = _wrap_list_n(link, ti.list_depth) - elif link: - display = link - elif ti.is_list: - display = _plain_list_type(base, ti.list_depth) - else: - display = f"`{base}`" - + display = _format_shape(field.shape, ctx, qualifiers) if not field.is_required: qualifiers.append("optional") - if qualifiers: return f"{display} ({', '.join(qualifiers)})" return display -def _linked_or_backticked(ti: TypeInfo, ctx: LinkContext | None) -> tuple[str, bool]: - """Return (formatted_string, has_link) for a TypeInfo component. +def _format_shape( + shape: FieldShape, ctx: LinkContext | None, qualifiers: list[str] +) -> str: + """Format a `FieldShape`, possibly appending qualifiers like `list`, `map`.""" + outer_depth, inner = _peel_arrays(shape) + + match inner: + case LiteralScalar(values=values): + if outer_depth > 0: + inside = " | ".join(f'"{v}"' for v in values) + return _plain_list_type(inside, outer_depth) + return _format_literal(values) + + case UnionRef(union=u): + if outer_depth > 0: + qualifiers.append("list") + return _format_union_members(u.members, ctx) + + case MapOf() as m: + map_str = _format_map(m, ctx) + if outer_depth > 0: + return _wrap_list_n(map_str.strip("`"), outer_depth) + return map_str + + case ModelRef() as m: + link = _model_link(m, ctx) + if outer_depth > 0: + return _wrap_list_n(link, outer_depth) + return link + + case NewTypeShape(name=name, ref=ref, inner=nt_inner): + link = resolve_type_link(TypeIdentity(ref, name), ctx) + if outer_depth > 0: + return _wrap_list_n(link, outer_depth) + if isinstance(nt_inner, ArrayOf): + qualifiers.append("list") + elif isinstance(nt_inner, MapOf): + qualifiers.append("map") + return link + + case Primitive() | AnyScalar() as s: + text, linked = _scalar_display(s, ctx) + if outer_depth > 0: + if linked: + return _wrap_list_n(text, outer_depth) + return _plain_list_type(text.strip("`"), outer_depth) + return text + + raise TypeError(f"Unhandled FieldShape: {shape!r}") + + +# ---- Underlying-type rendering for NewType pages ---- + + +def _peel_to_terminal(shape: FieldShape) -> FieldShape: + """Strip `NewTypeShape` / `ArrayOf` layers to find the terminal shape.""" + while True: + if isinstance(shape, NewTypeShape): + shape = shape.inner + elif isinstance(shape, ArrayOf): + shape = shape.element + else: + return shape - Used by format_underlying_type to decide whether container types - need broken-backtick formatting (interleaving backtick runs with - linked text). - When `has_link` is True, `formatted_string` is a markdown link - ready for broken-backtick container syntax. When False, it is a raw - name that the caller embeds inside backticks. - """ - identity = _linked_type_identity(ti) - if identity and ctx: - href = ctx.resolve_link(identity) - if href: - return _code_link(identity.name, href), True - return _markdown_type_name(ti), False +def format_underlying_type(shape: FieldShape, ctx: LinkContext | None = None) -> str: + """Format a NewType's underlying type for the page header, with links.""" + terminal = _peel_to_terminal(shape) + if isinstance(terminal, UnionRef): + return _format_union_members(terminal.union.members, ctx, separator=" | ") + if isinstance(terminal, MapOf): + return _format_map(terminal, ctx) -def format_underlying_type(ti: TypeInfo, ctx: LinkContext | None = None) -> str: - """Format a NewType's underlying type for the page header, with links. + # Link by the terminal primitive's identity, not the enclosing NewType's: + # this shape belongs to the NewType being rendered, so linking its own + # identity would self-link. The terminal's identity is always its + # underlying primitive (Geometry/BBox, a pydantic type, etc.). + identity: TypeIdentity | None = None + if isinstance(terminal, Primitive): + identity = _scalar_identity(terminal) - Links enums and models that have their own pages. Does not link the - outermost NewType (which would self-reference). Dict key/value types - use full link resolution since they reference other types. - """ - if ti.kind == TypeKind.UNION and ti.union_members: - return _format_union_members(ti.union_members, ctx, separator=" | ") - - if ti.is_dict and ti.dict_key_type and ti.dict_value_type: - key_str, key_linked = _linked_or_backticked(ti.dict_key_type, ctx) - val_str, val_linked = _linked_or_backticked(ti.dict_value_type, ctx) - if key_linked or val_linked: - if not key_linked: - key_str = f"`{key_str}`" - if not val_linked: - val_str = f"`{val_str}`" - return f"`map<`{key_str}`,`{val_str}`>`" - return f"`map<{key_str}, {val_str}>`" + depth, _ = _peel_arrays(shape) - # Only link enums and models -- skip is_semantic_newtype to avoid - # self-linking (this TypeInfo belongs to the NewType being rendered). - identity = ( - TypeIdentity.of(ti.source_type) - if ti.kind in (TypeKind.ENUM, TypeKind.MODEL) and ti.source_type - else None - ) if identity and ctx: href = ctx.resolve_link(identity) if href: linked = _code_link(identity.name, href) - if ti.is_list: - return _wrap_list_n(linked, ti.list_depth) + if depth > 0: + return _wrap_list_n(linked, depth) return linked - base = identity.name if identity else resolve_type_name(ti, "markdown") - if ti.is_list: - return _plain_list_type(base, ti.list_depth) + base = identity.name if identity else resolve_type_name(shape) + if depth > 0: + return _plain_list_type(base, depth) return f"`{base}`" diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/__init__.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/__init__.py new file mode 100644 index 000000000..3383f3a19 --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/__init__.py @@ -0,0 +1 @@ +"""PySpark codegen pipeline: ModelSpec to expression and test modules.""" diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_primitive_fill.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_primitive_fill.py new file mode 100644 index 000000000..17f1595e0 --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_primitive_fill.py @@ -0,0 +1,23 @@ +"""Shared fill-value table for non-string scalar Spark categories. + +Maps each SparkCategory that requires an explicit fill value to a +`(source_literal, runtime_value)` pair. The source literal is a valid +Python expression string; the runtime value is the corresponding Python +object. + +Consumers +--------- +- `constraint_dispatch._needs_explicit_fill`: category in PRIMITIVE_FILL_TABLE +- `test_renderer._fill_value_literal`: PRIMITIVE_FILL_TABLE[category][0] +- `test_data.base_row._primitive_default`: PRIMITIVE_FILL_TABLE[category][1] + +Adding a new numeric category here automatically wires it into all three. +""" + +from ..extraction.type_registry import SparkCategory + +PRIMITIVE_FILL_TABLE: dict[SparkCategory, tuple[str, object]] = { + "int": ("0", 0), + "float": ("0.0", 0.0), + "bool": ("False", False), +} diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_render_common.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_render_common.py new file mode 100644 index 000000000..bfa76ca0b --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_render_common.py @@ -0,0 +1,411 @@ +"""Shared rendering primitives used by `renderer` and `test_renderer`. + +Concerns: + +- `jinja_env` -- the cached Jinja2 environment. +- `py_literal` / `tuple_literal` -- render Python values back to source code. +- schema constant naming -- `schema_const_name` (the cross-module contract + between expression and test modules). +- check/label naming -- `check_name`, `field_label`, `column_level_suffix`, + `sanitize_field_name`, `COLUMN_LEVEL_FUNCTIONS` (membership), and + `_COLUMN_LEVEL_SUFFIXES` (label suffix lookup). +- emission rows -- `field_check_rows` and `model_check_rows` flatten a + check list into ordered rows carrying each row's final label, check + name, and (for field checks) disambiguated function name. The renderer + and test renderer both consume these rows, so the flatten-and-suffix + logic lives here once rather than in two positionally-coupled passes. + `disambiguate` (asymmetric, function-name keyed) and `_occurrence_indices` + (the shared collision primitive) back them. +""" + +from __future__ import annotations + +import functools +import re +from collections import Counter +from collections.abc import Hashable, Iterable +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import TypeVar + +from jinja2 import Environment, FileSystemLoader + +from overture.schema.system.field_path import ArrayPath, MapProjection + +from .check_ir import Check, ModelCheck +from .constraint_dispatch import ForbidIf, RequireIf, model_constraint_function + +__all__ = [ + "COLUMN_LEVEL_FUNCTIONS", + "FieldCheckRow", + "ModelCheckRow", + "check_name", + "column_level_suffix", + "disambiguate", + "field_check_rows", + "field_label", + "jinja_env", + "map_runtime_helper", + "model_check_rows", + "py_literal", + "sanitize_field_name", + "schema_const_name", + "tuple_literal", +] + +_K = TypeVar("_K", bound=Hashable) + +# Constraint functions that emit a column-level check (one per field +# rather than per element), used by the check builder to split them +# into their own `Check` IR nodes. +COLUMN_LEVEL_FUNCTIONS: frozenset[str] = frozenset( + { + "check_required", + "check_array_min_length", + "check_array_max_length", + "check_struct_unique", + } +) + +# Violation label suffix per column-level check that shares its +# field's structural path. `check_required` lands on its field's own +# path, so it stays absent from this table. +_COLUMN_LEVEL_SUFFIXES: dict[str, str] = { + "check_array_min_length": "_min_length", + "check_array_max_length": "_max_length", + "check_struct_unique": "_unique", +} + +_MAP_RUNTIME_HELPERS: dict[MapProjection, str] = { + MapProjection.KEY: "map_keys_check", + MapProjection.VALUE: "map_values_check", +} + + +def map_runtime_helper(projection: MapProjection) -> str: + """PySpark column-patterns helper name for a map projection. + + `MapProjection.KEY` -> `map_keys_check`; + `MapProjection.VALUE` -> `map_values_check`. This is a pyspark-layer + concern; the mapping lives here rather than on `MapProjection` itself + (a system-package enum) to avoid a layering violation. + """ + return _MAP_RUNTIME_HELPERS[projection] + + +_TEMPLATES_DIR = Path(__file__).parent / "templates" + + +@functools.lru_cache(maxsize=1) +def jinja_env() -> Environment: + """Return the shared Jinja2 environment for PySpark code generation templates.""" + env = Environment( + loader=FileSystemLoader(_TEMPLATES_DIR), + trim_blocks=True, + lstrip_blocks=True, + keep_trailing_newline=True, + autoescape=False, + ) + env.filters["py_literal"] = py_literal + return env + + +def schema_const_name(model_name: str) -> str: + """Name of the generated `MODELNAME_SCHEMA` StructType constant. + + A cross-module contract: the generated test module imports this + constant by name from the generated expression module. + """ + return f"{model_name.upper()}_SCHEMA" + + +_CHECK_PREFIX = "check_" + + +def tuple_literal(rendered_items: Iterable[str]) -> str: + """Wrap pre-rendered items as a Python tuple literal source. + + A single-element tuple needs a trailing comma; this helper applies + that rule so callers rendering enum-like values that don't fit + `py_literal` can still share its tuple-formatting behaviour. + """ + items = list(rendered_items) + joined = ", ".join(items) + return f"({joined},)" if len(items) == 1 else f"({joined})" + + +def py_literal(value: object) -> str: + """Render a Python value as source code. + + Recurses into containers to extract `Enum.value` (since `repr()` of + an Enum member is not valid Python). Quote style and line wrapping + are left to `ruff format`. + """ + if isinstance(value, Enum): + return py_literal(value.value) + if isinstance(value, dict): + items = ", ".join(f"{py_literal(k)}: {py_literal(v)}" for k, v in value.items()) + return "{" + items + "}" + if isinstance(value, list): + return "[" + ", ".join(py_literal(v) for v in value) + "]" + if isinstance(value, tuple): + return tuple_literal(py_literal(v) for v in value) + if isinstance(value, frozenset): + if not value: + return "frozenset()" + # Sort the rendered items so regenerated source is stable across runs + # (set iteration order is not). + return "frozenset({" + ", ".join(sorted(py_literal(v) for v in value)) + "})" + return repr(value) + + +def check_name(function: str, override: str | None = None) -> str: + """Strip the `check_` prefix to produce a human-readable check name.""" + if override is not None: + return override + return function.removeprefix(_CHECK_PREFIX) + + +# Collapses runs of path punctuation (`.`, `[`, `]`, `{`, `}`, `_`) to a +# single `_` for identifier sanitization (e.g. `names.common{key}` -> +# `names_common_key`). +_PATH_SEPARATOR_RUN = re.compile(r"[.\[\]{}_]+") + + +def sanitize_field_name(field: str) -> str: + """Convert an encoded field-path string to a valid Python identifier fragment.""" + return _PATH_SEPARATOR_RUN.sub("_", field).strip("_") + + +def column_level_suffix(check: Check) -> str: + """Return the column-level label suffix for `check`, or empty string. + + Column-level checks (`check_array_min_length`, `check_struct_unique`, + etc.) share their structural path with the field they constrain; the + suffix differentiates the violation label so each check reports a + distinct `Check.field`. + """ + if not check.descriptors: + return "" + return _COLUMN_LEVEL_SUFFIXES.get(check.descriptors[0].function, "") + + +def field_label(check: Check) -> str: + """Render the violation label for a Check. + + Combines the structural field path with any column-level suffix + (`_min_length`, `_unique`, etc.) so each check reports a distinct + `Check.field` even when several share a structural path. + """ + return f"{check.target}{column_level_suffix(check)}" + + +def _model_check_base_label(check: ModelCheck) -> str: + """Compute the violation field label sans collision suffix. + + - `require_if` / `forbid_if` produce a per-target label + (`field_required` / `path.field_forbidden`); each descriptor + carries a single target field (multi-field decorators split at + dispatch time). + - Other kinds (`require_any_of`, `radio_group`, `min_fields_set`) + name the whole constraint; on `ArrayPath` targets they use the + path itself so anchors are distinguishable across nestings. + """ + match check.descriptor: + case RequireIf(): + kind_suffix = "_required" + case ForbidIf(): + kind_suffix = "_forbidden" + case _: + if isinstance(check.target, ArrayPath): + return str(check.target) + return check_name(model_constraint_function(check.descriptor)) + target = check.descriptor.field_names[0] + if not isinstance(check.target, ArrayPath): + return f"{target}{kind_suffix}" + return f"{check.target}.{target}{kind_suffix}" + + +def _occurrence_indices(keys: list[_K]) -> list[tuple[int, int]]: + """Pair each key with `(occurrence_index, total_count)`. + + `occurrence_index` is the 0-based position of the key among its + equal siblings; `total_count` is how many times the key appears in + `keys`. Both collision styles -- `disambiguate` (function names) and + the symmetric label suffixing in the row builders -- need this + "where am I within my collision group" view. + """ + counts: Counter[_K] = Counter(keys) + seen: Counter[_K] = Counter() + result: list[tuple[int, int]] = [] + for key in keys: + result.append((seen[key], counts[key])) + seen[key] += 1 + return result + + +def disambiguate(names: list[str]) -> list[str]: + """Make a list of names unique by appending `_N` to repeated entries. + + The first occurrence of a name is left bare; the second becomes + `name_1`, the third `name_2`, and so on. Names that appear once are + untouched. This is the asymmetric style, keyed on the function-name + string: leaving the first occurrence bare keeps readable identifiers + for the common no-collision case. + + Assumes no input name already matches a generated `name_N` form; a + collision there would reintroduce a duplicate. Field names in + practice never carry that suffix, so the assumption holds. + """ + return [ + f"{name}_{idx}" if total > 1 and idx > 0 else name + for name, (idx, total) in zip(names, _occurrence_indices(names), strict=True) + ] + + +def _symmetric_label_suffixes(keys: list[_K]) -> list[str]: + """Per-key violation-label collision suffixes, symmetric across a group. + + Every member of a colliding group receives a `_N` suffix including + the first (`_0`, `_1`, ...); unique keys stay bare. Symmetric unlike + `disambiguate` because violation labels in a colliding group all + share the same base name, so each needs an explicit index to stay a + distinct `Check.field` identity (which keys `suppress` matching, + `explain_errors` metadata, and the test's `expected_field`). + """ + return [f"_{idx}" if total > 1 else "" for idx, total in _occurrence_indices(keys)] + + +@dataclass(frozen=True, slots=True) +class FieldCheckRow: + """One emitted field-check row, with its final derived strings. + + The renderer emits one row per descriptor of each `Check`. + `field_check_rows` flattens the check list into these rows once, + computing both the symmetric `label` collision suffix and the + asymmetric `func_name` disambiguation, so the renderer and test + renderer agree without each re-deriving them by a positional index. + + Attributes + ---------- + check + The originating field check. + descriptor_idx + Index of this row's descriptor within `check.descriptors`. + label + The violation `field=` label, including any collision suffix. + name + The check name (`check_name(desc.function, desc.check_name)`). + func_name + The disambiguated private `_..._check` function name. + """ + + check: Check + descriptor_idx: int + label: str + name: str + func_name: str + + +def field_check_rows(field_checks: list[Check]) -> list[FieldCheckRow]: + """Flatten field checks into emission rows with final derived strings. + + Computes both collision passes over the *unfiltered* list, so the + expression module (rendered once across every arm) and a per-arm test + module agree: a per-arm subset could otherwise hide a collision the + shared module still carries, emitting an `expected_field` the module + never produces. Callers filter the returned rows to an arm afterward + rather than computing suffixes over a subset. + + Parameters + ---------- + field_checks + The complete field-check list for one generated module, before + any per-arm filtering. + + Returns + ------- + list + One `FieldCheckRow` per emitted `(check, descriptor)`, in + flattened emission order. + """ + flattened: list[tuple[Check, int, str, str]] = [] + raw_func_names: list[str] = [] + for check in field_checks: + label = field_label(check) + multi = len(check.descriptors) > 1 + for desc_idx, desc in enumerate(check.descriptors): + name = check_name(desc.function, desc.check_name) + func_suffix = f"_{name}" if multi else "" + raw_func_names.append(f"_{sanitize_field_name(label)}{func_suffix}_check") + flattened.append((check, desc_idx, label, name)) + func_names = disambiguate(raw_func_names) + label_suffixes = _symmetric_label_suffixes( + [(label, name) for _check, _idx, label, name in flattened] + ) + return [ + FieldCheckRow(check, desc_idx, f"{label}{label_suffix}", name, func_name) + for (check, desc_idx, label, name), label_suffix, func_name in zip( + flattened, label_suffixes, func_names, strict=True + ) + ] + + +@dataclass(frozen=True, slots=True) +class ModelCheckRow: + """One emitted model-check row, with its final derived strings. + + Model function names embed `idx` and are unique by construction, so + a row carries no `func_name` -- the renderer builds it from `idx`. + + Attributes + ---------- + check + The originating model check. + idx + Position of this check in the unfiltered model-check list; the + renderer embeds it in the private function name. + label + The violation `field=` label, including any collision suffix. + name + The check name (`check_name(model_constraint_function(...))`). + """ + + check: ModelCheck + idx: int + label: str + name: str + + +def model_check_rows(model_checks: list[ModelCheck]) -> list[ModelCheckRow]: + """Flatten model checks into emission rows with final derived strings. + + Like `field_check_rows`, label collision suffixes are computed over + the *unfiltered* list so the expression module and per-arm test + modules agree; callers filter the returned rows to an arm afterward. + + Parameters + ---------- + model_checks + The complete model-check list for one generated module, before + any per-arm filtering. + + Returns + ------- + list + One `ModelCheckRow` per model check, in list order. + """ + base_labels = [_model_check_base_label(check) for check in model_checks] + label_suffixes = _symmetric_label_suffixes(base_labels) + return [ + ModelCheckRow( + check, + idx, + f"{base_label}{label_suffix}", + check_name(model_constraint_function(check.descriptor)), + ) + for idx, (check, base_label, label_suffix) in enumerate( + zip(model_checks, base_labels, label_suffixes, strict=True) + ) + ] diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_builder.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_builder.py new file mode 100644 index 000000000..56ab5499c --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_builder.py @@ -0,0 +1,949 @@ +"""Walk FieldSpec trees to produce Check/ModelCheck IR for rendering. + +Consults the constraint dispatch table to map each constraint to a +descriptor, then applies composition rules the dispatch table can't see: + +- Coalesce ordering: gather descriptors for the same field into one + `Check` (required first, then enum, then dispatched constraints), + deduplicate, and split column-level checks into separate suffixed checks. +- Target resolution: a shape walker descends each field's `FieldShape` + tree, building the `ScalarPath` or `ArrayPath` target by appending + segments as it goes -- so the path read in the code is the path that + lands in the IR. Entering a `list[...]` layer promotes the path's + terminal struct segment to an iterated `ArraySegment`. +- Subtype gating: annotate variant-specific fields with discriminator + `Guard`s, synthesize forbid_if/require_if for absent or required + variants, and gate check_required under nullable struct ancestors. +""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass, replace + +from pydantic import BaseModel +from typing_extensions import assert_never + +from overture.schema.system.field_path import ( + ArrayPath, + ArraySegment, + FieldPath, + MapPath, + MapProjection, + MapSegment, + ScalarPath, + promote_terminal_array, + promote_terminal_map, +) +from overture.schema.system.model_constraint import ( + FieldEqCondition, + ModelConstraint, + Not, +) + +from ..extraction.field import ( + AnyScalar, + ArrayOf, + ConstraintSource, + FieldShape, + LiteralScalar, + MapOf, + ModelRef, + NewTypeShape, + Primitive, + Scalar, + UnionRef, +) +from ..extraction.field_walk import ( + all_constraints, + enum_source, + has_array_layer, + terminal_of, + terminal_primitive, + terminal_scalar, +) +from ..extraction.literal_alternatives import LiteralAlternatives +from ..extraction.specs import FieldSpec, ModelSpec, RecordSpec, UnionSpec +from ..extraction.type_registry import PRIMITIVE_TYPES +from ._render_common import COLUMN_LEVEL_FUNCTIONS +from .check_ir import ( + Check, + ColumnGuard, + ElementGuard, + Guard, + ModelCheck, +) +from .constraint_dispatch import ( + ExpressionDescriptor, + ForbidIf, + RequireIf, + dispatch_base_type, + dispatch_constraint, + dispatch_model_constraint, + dispatch_newtype, + forbid_if_field_shapes, +) + +__all__ = [ + "build_checks", +] + + +def _dispatch_layer_constraints( + constraints: tuple[ConstraintSource, ...], + base_type: str | None, +) -> list[ExpressionDescriptor]: + """Dispatch one shape layer's constraints, skipping primitive-inherent ones.""" + descriptors: list[ExpressionDescriptor] = [] + for cs in constraints: + if cs.source_name is not None and cs.source_name in PRIMITIVE_TYPES: + continue + desc = dispatch_constraint(cs.constraint, base_type=base_type) + if desc is not None: + descriptors.append(desc) + return descriptors + + +def _literal_alternatives(shape: Scalar | MapOf) -> tuple[object, ...]: + """Return the allowed literal values from a `LiteralAlternatives` constraint, or `()`. + + `MapOf` returns `()`: a map column carries no literal alternative of its + own (its key/value projections reach their own `Scalar` shapes, which are + handled there). + """ + if isinstance(shape, MapOf): + return () + for cs in shape.constraints: + if isinstance(cs.constraint, LiteralAlternatives): + return cs.constraint.values + return () + + +def _apply_literal_bypass( + descriptors: list[ExpressionDescriptor], + allow_literals: tuple[object, ...], +) -> list[ExpressionDescriptor]: + """Stamp `allow_literals` onto each content descriptor. + + Content descriptors are the non-required checks: enum, pattern, + bounds, base-type checks. `check_required` is excluded by callers + who pass only the content portion of the descriptor list. + """ + if not allow_literals: + return descriptors + return [replace(desc, allow_literals=allow_literals) for desc in descriptors] + + +def _enum_values(scalar: Scalar) -> list[object] | None: + """Return enum/literal values for a terminal `Scalar`, or `None`.""" + if isinstance(scalar, LiteralScalar): + return list(scalar.values) + src = enum_source(scalar) + if src is not None: + return [m.value for m in src] + return None + + +def _required_descriptor(gate: FieldPath | None) -> ExpressionDescriptor: + return ExpressionDescriptor(function="check_required", gate=gate) + + +@dataclass(frozen=True, slots=True) +class _ShapeTerminal: + """A `ModelRef`/`UnionRef` terminal and the path the walker reached it at. + + The `FieldSpec` recursion uses `path` directly as the prefix for the + sub-model's or sub-union's fields. The walker returns `None` instead + of a `_ShapeTerminal` for terminals it fully handles itself (scalars, + maps, and NewTypes with a dispatch override). + """ + + ref: ModelRef | UnionRef + path: FieldPath + + +def _walk_field_shape( + shape: FieldShape, + path: FieldPath, + *, + base_type: str | None, + required: bool, + required_gate: FieldPath | None, + carried_element: list[ExpressionDescriptor], +) -> tuple[list[Check], _ShapeTerminal | None]: + """Descend a `FieldShape`, emitting the field's own Checks. + + Builds the `FieldPath` target structurally: `ArrayOf` promotes the + path's terminal segment, `NewTypeShape` passes the path through, + terminals emit at the path reached. Returns the emitted Checks plus, + at a `ModelRef`/`UnionRef` terminal, a `_ShapeTerminal` for the + `FieldSpec` recursion (`None` for terminals the walker fully handles). + + Parameters + ---------- + path + The path reached so far, promoted once per `ArrayOf` layer + crossed. `required` and `path` move together: a field's path + starts as a plain struct path and is promoted exactly when the + first `ArrayOf` clears `required`, so while `required` holds the + path is still the plain struct path -- a standalone + `check_required` always lands there. + required + Whether the field still needs a `check_required`. Cleared by the + first `ArrayOf`: before it, `check_required` merges into the + terminal Check; from it on, it is a standalone column-level Check. + carried_element + Element-level descriptors from `ArrayOf` layers above, prepended + to the terminal's own element-level descriptors. + """ + match shape: + case NewTypeShape(name=name, inner=inner): + nt_descriptors = dispatch_newtype(name) + if nt_descriptors is not None: + if isinstance(path.segments[-1], ArraySegment): + # A NewType with a dispatch override nested under a list + # layer has no schema field; raise to keep the gap loud + # rather than emit an untested target (cf. list[list[Union]]). + raise NotImplementedError( + f"NewType with a dispatch override ({name}) nested " + "under a list layer is not supported" + ) + descriptors = list(nt_descriptors) + if required: + descriptors.insert(0, _required_descriptor(required_gate)) + return [Check(descriptors=tuple(descriptors), target=path)], None + return _walk_field_shape( + inner, + path, + base_type=base_type, + required=required, + required_gate=required_gate, + carried_element=carried_element, + ) + + case ArrayOf(element=element, constraints=constraints): + layer_descriptors = _dispatch_layer_constraints( + constraints, + base_type, + ) + column_descriptors = list( + dict.fromkeys( + d for d in layer_descriptors if d.function in COLUMN_LEVEL_FUNCTIONS + ) + ) + element_descriptors = [ + d for d in layer_descriptors if d.function not in COLUMN_LEVEL_FUNCTIONS + ] + checks: list[Check] = [] + if required: + checks.append( + Check( + descriptors=(_required_descriptor(required_gate),), + target=path, + ) + ) + checks.extend( + Check(descriptors=(d,), target=path) for d in column_descriptors + ) + sub_checks, terminal = _walk_field_shape( + element, + promote_terminal_array(path), + base_type=base_type, + required=False, + required_gate=required_gate, + carried_element=[*carried_element, *element_descriptors], + ) + return [*checks, *sub_checks], terminal + + case UnionRef(): + terminal_seg = path.segments[-1] + if isinstance(terminal_seg, ArraySegment) and terminal_seg.iter_count > 1: + # `list[list[Union]]` would build a multi-iter union target, + # but no schema field has that shape. The walker raises to + # keep the gap loud rather than silently emit one. + raise NotImplementedError( + "Union nested under multiple list layers " + "(list[list[Union]]) is not supported" + ) + return _ref_terminal_checks(shape, path, required, required_gate) + + case ModelRef(): + return _ref_terminal_checks(shape, path, required, required_gate) + + case Primitive() | LiteralScalar() | AnyScalar(): + return _terminal_scalar_checks( + shape, + path, + base_type=base_type, + required=required, + required_gate=required_gate, + carried_element=carried_element, + ), None + + case MapOf(key=key_shape, value=value_shape): + # A map is itself a terminal column: its own value carries the + # required check and any map-level constraints (currently always + # empty -- map-level length constraints are rejected at + # extraction). The key and value layers are walked separately so + # their per-key/per-value constraints land on `MapPath` targets. + # A `ModelRef`/`UnionRef` projection hands back a `_ShapeTerminal` + # for the caller to descend into, exactly as a `list[Model]` + # element does. + field_checks = _terminal_scalar_checks( + shape, + path, + base_type=base_type, + required=required, + required_gate=required_gate, + carried_element=carried_element, + ) + key_checks, key_terminal = _map_projection_checks( + key_shape, path, MapProjection.KEY + ) + value_checks, value_terminal = _map_projection_checks( + value_shape, path, MapProjection.VALUE + ) + if key_terminal is not None and value_terminal is not None: + raise NotImplementedError( + "map with a model key and a model value is not supported" + ) + terminal = value_terminal if value_terminal is not None else key_terminal + return [*field_checks, *key_checks, *value_checks], terminal + + assert_never(shape) + + +def _terminal_scalar_checks( + shape: Scalar | MapOf, + path: FieldPath, + *, + base_type: str | None, + required: bool, + required_gate: FieldPath | None, + carried_element: list[ExpressionDescriptor], +) -> list[Check]: + """Build the Check(s) for a terminal value: enum, constraints, base type. + + Shared by the scalar-terminal arm and a map field's own value -- a + `MapOf` is itself a terminal column, distinct from its key/value layers. + """ + element_descriptors = list(carried_element) + enum_values = _enum_values(shape) if isinstance(shape, Scalar) else None + if enum_values is not None: + element_descriptors.append( + ExpressionDescriptor(function="check_enum", args=(tuple(enum_values),)) + ) + element_descriptors.extend( + _dispatch_layer_constraints(shape.constraints, base_type) + ) + if base_type is not None: + base_descriptors = dispatch_base_type(base_type) + if base_descriptors is not None: + element_descriptors.extend(base_descriptors) + element_descriptors = list(dict.fromkeys(element_descriptors)) + element_descriptors = _apply_literal_bypass( + element_descriptors, _literal_alternatives(shape) + ) + + if required: + return [ + Check( + descriptors=(_required_descriptor(required_gate), *element_descriptors), + target=path, + ) + ] + if element_descriptors: + return [Check(descriptors=tuple(element_descriptors), target=path)] + return [] + + +@dataclass(frozen=True) +class MapProjectionVerdict: + """Whether a map's projected key/value shape is representable as a `MapPath`. + + `reason` names why an unrepresentable shape was rejected (for the + `NotImplementedError` message); it is `None` when `representable` is True. + `has_value_to_validate` reports whether the projected shape carries a + constraint or descends into a model -- the loud/quiet discriminator: an + unrepresentable shape with something to validate raises, an unrepresentable + shape with nothing to validate is silently dropped. + """ + + representable: bool + reason: str | None + has_value_to_validate: bool + + +def classify_map_projection( + sub_shape: FieldShape, + map_path: FieldPath, +) -> MapProjectionVerdict: + """Classify a map's projected key/value shape against the representable bound. + + The single source of truth for which map projections a `MapPath` can + locate. The representable shape: a scalar terminal or `ModelRef`/`UnionRef` + terminal, reached WITHOUT array iteration (`map_path` is not an + `ArrayPath`), with no `ArrayOf` layer in the projected shape. Both + `_map_projection_checks` (shape-level) and the path-level guards in + `field_path.py` (`promote_terminal_map` rejecting an `ArrayPath`) enforce + this boundary; this classifier states it once so the prohibitions agree by + construction rather than by parallel maintenance. + + Two shapes fall outside the bound and have no `MapPath`: + + - a map reached through an array (`list[dict[K, V]]`, a `map_path` that is + an `ArrayPath`), whose key/value can't anchor a struct-prefixed `MapPath`; + - a key/value carrying an array layer (`dict[K, list[V]]`), whose scalar + terminal sits under an `ArrayOf` that `terminal_scalar` would unwrap. + """ + is_ref_terminal = isinstance(terminal_of(sub_shape), (ModelRef, UnionRef)) + has_value_to_validate = bool(all_constraints(sub_shape)) or is_ref_terminal + if isinstance(map_path, ArrayPath): + return MapProjectionVerdict( + representable=False, + reason="map reached through an array is not representable", + has_value_to_validate=has_value_to_validate, + ) + if has_array_layer(sub_shape): + return MapProjectionVerdict( + representable=False, + reason="map value carrying a list layer (dict[K, list[V]]) is not representable", + has_value_to_validate=has_value_to_validate, + ) + if not is_ref_terminal and terminal_scalar(sub_shape) is None: + return MapProjectionVerdict( + representable=False, + reason="constraint on a non-scalar terminal", + has_value_to_validate=has_value_to_validate, + ) + return MapProjectionVerdict( + representable=True, reason=None, has_value_to_validate=has_value_to_validate + ) + + +def _map_projection_checks( + sub_shape: FieldShape, + map_path: FieldPath, + projection: MapProjection, +) -> tuple[list[Check], _ShapeTerminal | None]: + """Walk a map's key or value shape, emitting checks on a `MapPath` target. + + Supports two shapes reached without array iteration: a scalar terminal + (`dict[K, scalar]` -- per-key/value constraints land on a bare `MapPath`) + and a `ModelRef`/`UnionRef` terminal (`dict[K, Model]` -- the returned + `_ShapeTerminal` lets the caller descend into the model's fields and + constraints on a `MapPath` leaf, mirroring a `list[Model]` element). + + `classify_map_projection` is the arbiter of which shapes are + representable. An unrepresentable shape carrying something to validate + (`has_value_to_validate`) raises `NotImplementedError` to keep the dropped + check loud; an unrepresentable shape with nothing to validate yields no + checks. The constraint -- not the shape alone -- is what stays loud, + matching the silent treatment of unconstrained maps. + """ + verdict = classify_map_projection(sub_shape, map_path) + if not verdict.representable: + if verdict.has_value_to_validate: + raise NotImplementedError( + f"map {projection.value} on an unsupported shape " + f"({verdict.reason}) is not supported ({sub_shape!r})" + ) + return [], None + primitive = terminal_primitive(sub_shape) + sub_checks, terminal = _walk_field_shape( + sub_shape, + promote_terminal_map(map_path, projection), + base_type=primitive.base_type if primitive is not None else None, + required=False, + required_gate=None, + carried_element=[], + ) + return sub_checks, terminal + + +def _ref_terminal_checks( + ref: ModelRef | UnionRef, + path: FieldPath, + required: bool, + required_gate: FieldPath | None, +) -> tuple[list[Check], _ShapeTerminal]: + """Handle a `ModelRef`/`UnionRef` terminal: emit `check_required`, hand back the ref. + + A required model or union field always gets a standalone + `check_required` Check; `required` holds only before any `ArrayOf`, + so `path` is the field's plain struct path. The sub-fields are the + caller's job, reached via the returned `_ShapeTerminal`. + """ + checks: list[Check] = [] + if required: + checks.append( + Check( + descriptors=(_required_descriptor(required_gate),), + target=path, + ) + ) + return checks, _ShapeTerminal(ref=ref, path=path) + + +def _build_field_checks( + field_spec: FieldSpec, + prefix: FieldPath = ScalarPath(), + *, + nullable_gate: FieldPath | None = None, + arm: str | None = None, +) -> tuple[list[Check], list[ModelCheck]]: + """Build Checks for a single field by walking its shape tree. + + `arm` is the singleton union-arm discriminator value the field belongs + to (when it lives in exactly one arm), or `None` when the field is + shared. It propagates to any model constraints discovered through this + field's sub-models so per-arm test modules can filter them correctly. + """ + # `prefix` is a ScalarPath/ArrayPath, or a MapPath when descending into + # a `dict[K, Model]` value model -- all three define `append_struct`, + # which extends the path's struct leaf with this field's name. + path = prefix.append_struct(field_spec.name) + checks, terminal = _walk_field_shape( + field_spec.shape, + path, + base_type=( + p.base_type + if (p := terminal_primitive(field_spec.shape)) is not None + else None + ), + required=field_spec.is_required, + required_gate=nullable_gate, + carried_element=[], + ) + + model_checks: list[ModelCheck] = [] + match terminal: + case None: + pass + case _ShapeTerminal(ref=UnionRef(union=union_spec), path=terminal_path): + sub_field_checks, sub_model_checks = _recurse_into_union( + union_spec, terminal_path, arm=arm + ) + checks.extend(sub_field_checks) + model_checks.extend(sub_model_checks) + case _ShapeTerminal(ref=ModelRef(model=model_spec), path=terminal_path): + sub_field_checks, sub_model_checks = _recurse_into_model( + model_spec, + terminal_path, + field_spec.is_optional, + nullable_gate, + arm=arm, + ) + checks.extend(sub_field_checks) + model_checks.extend(sub_model_checks) + case _ShapeTerminal(ref=ref): + raise AssertionError( + f"unhandled _ShapeTerminal.ref variant: {type(ref).__name__}" + ) + + return checks, model_checks + + +def _recurse_into_model( + model_spec: RecordSpec, + prefix: FieldPath = ScalarPath(), + is_optional: bool = False, + nullable_gate: FieldPath | None = None, + *, + arm: str | None = None, +) -> tuple[list[Check], list[ModelCheck]]: + """Walk a MODEL-kind field's children plus its model-level constraints. + + `prefix` is the terminal path the shape walker reached the `ModelRef` + at, defaulting to the empty `ScalarPath()` at the row root. Its terminal + segment is an `ArraySegment` (the field is a list) or a `MapSegment` + (the field is a `dict[K, Model]` reached through its key/value + projection) exactly when the model is reached through iteration, which + resets the nullable gate (the iteration itself handles per-element + nullability). + + `arm` propagates from the union arm whose variant-specific field led + here, so model constraints declared on the sub-model are tagged with + that arm rather than `None` (which would route them to every per-arm + test). + """ + last_seg = prefix.segments[-1] if prefix.segments else None + field_is_iterated = isinstance(last_seg, (ArraySegment, MapSegment)) + if field_is_iterated: + child_gate: FieldPath | None = None + else: + child_gate = prefix if is_optional else nullable_gate + + field_checks: list[Check] = [] + model_checks: list[ModelCheck] = [] + for sub_field in model_spec.fields: + sub_field_checks, sub_model_checks = _build_field_checks( + sub_field, + prefix=prefix, + nullable_gate=child_gate, + arm=arm, + ) + field_checks.extend(sub_field_checks) + model_checks.extend(sub_model_checks) + + if model_spec.constraints: + constraint_gate = ( + prefix + if is_optional and not field_is_iterated and isinstance(prefix, ArrayPath) + else None + ) + sub_model_constraint_checks = _dispatch_model_constraints( + model_spec.constraints, + model_spec.fields, + target=_model_constraint_target(prefix), + arm=arm, + gate=constraint_gate, + ) + if sub_model_constraint_checks: + _guard_struct_nested_anchor(prefix, model_spec.name) + model_checks.extend(sub_model_constraint_checks) + return field_checks, model_checks + + +def _is_struct_only_prefix(prefix: FieldPath) -> bool: + """Non-root struct path with no array traversal. + + True when `prefix` has one or more struct segments but no array + iteration -- meaning discriminator column access and model-constraint + targeting cannot use the prefix without resolving it into a + struct-qualified path, which the current renderer does not support. + """ + return not isinstance(prefix, ArrayPath) and bool(prefix.segments) + + +def _reject_struct_only_prefix(prefix: FieldPath, message: str) -> None: + """Raise `NotImplementedError(message)` when `prefix` is struct-only. + + Shared mechanism for the struct-nested guards: the renderer supports + neither model-constraint anchoring nor column-level discriminator + gating at a struct-only prefix, so reaching one with a real check is a + renderer gap rather than a normal case. + """ + if _is_struct_only_prefix(prefix): + raise NotImplementedError(message) + + +def _guard_struct_nested_anchor(prefix: FieldPath, name: str) -> None: + """Raise when emitting a model constraint at a struct-only prefix. + + See `_model_constraint_target`: in that case the constraint's target + collapses to the row root, which is wrong for any non-skipped + constraint. Today only `NoExtraFieldsConstraint` reaches here (and + dispatches to None); a real descriptor at this depth is a renderer + gap, not a normal case. A `MapPath` is exempt -- it is a valid anchor + (`_model_constraint_target` keeps it, and the renderer wraps the check + in `map_values_check`/`map_keys_check`). + """ + if isinstance(prefix, MapPath): + return + _reject_struct_only_prefix( + prefix, + f"Model constraint on struct-nested {name!r} " + f"(reached at {prefix!r}) -- the renderer has no anchor " + "for nested-struct model constraints.", + ) + + +def _guard_struct_nested_variant_fields(prefix: FieldPath, name: str) -> None: + """Raise when emitting variant-gated field checks at a struct-only prefix. + + A `ColumnGuard` carries a bare discriminator name that renders as + `F.col("")` -- a top-level column access. When the + union is reached through a plain struct field, the discriminator lives + at `.`, so the rendered gate reads the wrong + column. Raising loudly is safer than emitting a mis-gated check; no + current schema nests a discriminated union under a plain struct. + """ + _reject_struct_only_prefix( + prefix, + f"Discriminated union {name!r} with variant-gated field checks " + f"at struct-nested prefix {prefix!r} -- `ColumnGuard` would " + "render the discriminator as a top-level column, not a " + "struct-qualified path.", + ) + + +def _recurse_into_union( + union_spec: UnionSpec, + prefix: FieldPath = ScalarPath(), + *, + arm: str | None = None, +) -> tuple[list[Check], list[ModelCheck]]: + """Walk a UNION-kind field's variants, gathering Checks and ModelChecks. + + `prefix` is the terminal path the shape walker reached the `UnionRef` + at; the union's variant fields live directly under it. An `ArrayPath` + prefix means the union is reached through array iteration, so variant + gates are element-level and model constraints target that path. + + `arm` is the outer union arm whose variant-specific field reached this + inner union. It tags any model constraints discovered here so they + aren't propagated to other arms' test modules. + """ + mapping = union_spec.discriminator_mapping or {} + value_by_class = {cls: value for value, cls in mapping.items()} + union_target = _model_constraint_target(prefix) + + field_checks, field_model_checks = _field_checks_for_union( + union_spec, value_by_class, prefix=prefix, arm=arm + ) + union_level_checks = _model_checks_for_union( + union_spec, value_by_class, union_target, arm=arm + ) + exclusivity_checks = _exclusivity_checks_for_union( + union_spec, value_by_class, union_target, arm=arm + ) + if union_level_checks or exclusivity_checks: + _guard_struct_nested_anchor(prefix, union_spec.name) + return field_checks, union_level_checks + field_model_checks + exclusivity_checks + + +def _model_constraint_target(prefix: FieldPath) -> FieldPath: + """Where a model constraint's check should be anchored. + + Three supported cases: + + - `ArrayPath` -- constraints on a sub-model reached through array + iteration target the array path (so the renderer wraps the check + in `array_check`). + - `MapPath` -- constraints on a `dict[K, Model]` value model target the + map path (so the renderer wraps the check in `map_values_check`), + mirroring the array case. + - Empty or struct-only `ScalarPath` -- constraints anchor at the row + root. Pure struct nesting (e.g. `Names` reached at + `ScalarPath('names')`) collapses here because the renderer has no + anchor for nested-struct model constraints. The only constraint kind + currently reachable through pure struct nesting is + `NoExtraFieldsConstraint`, which `dispatch_model_constraint` + discards before the target is consulted, so the collapse is + observationally inert today; a non-skipped constraint at this depth + would surface as a wrong-anchor bug. + """ + return prefix if isinstance(prefix, (ArrayPath, MapPath)) else ScalarPath() + + +def _dispatch_model_constraints( + constraints: tuple[ModelConstraint, ...], + fields: list[FieldSpec], + *, + target: FieldPath = ScalarPath(), + arm: str | None = None, + gate: FieldPath | None = None, +) -> list[ModelCheck]: + """Dispatch model constraints to ModelChecks.""" + return [ + ModelCheck(descriptor=desc, target=target, arm=arm, gate=gate) + for mc in constraints + for desc in dispatch_model_constraint(mc, fields) + ] + + +def _singleton_arm(values: tuple[str, ...]) -> str | None: + """Return the sole arm in `values`, or None when there isn't exactly one. + + No real schema today has a variant-specific field belonging to a + proper subset of arms (2-of-N): every variant-specific field is + declared on exactly one arm. If a future schema introduces a 2-of-N + field whose sub-model declares model constraints, this collapse + would broadcast those constraints to every arm (including the ones + the field doesn't belong to). `TestMultiArmVariantSourcesPolicy` + pins the current behaviour as a tombstone. + """ + return values[0] if len(values) == 1 else None + + +def _field_checks_for_union( + spec: UnionSpec, + value_by_class: dict[type[BaseModel], str], + prefix: FieldPath = ScalarPath(), + *, + arm: str | None = None, +) -> tuple[list[Check], list[ModelCheck]]: + """Build field checks for a union spec's annotated fields. + + `arm` is the outer-union arm threaded through from an enclosing + `_recurse_into_union`. When present, every sub-model constraint + reached from here inherits that arm -- the inner union's own + discriminator is irrelevant to per-arm test filtering, which always + keys on the outermost union's discriminator. + """ + guard_cls: type[Guard] = ( + ElementGuard if isinstance(prefix, ArrayPath) else ColumnGuard + ) + field_checks: list[Check] = [] + model_checks: list[ModelCheck] = [] + discriminator = spec.discriminator_field + for af in spec.annotated_fields: + values: tuple[str, ...] = () + if af.variant_sources is not None and discriminator is not None: + values = tuple( + value_by_class[src] + for src in af.variant_sources + if src in value_by_class + ) + # Outer arm dominates: when this is a nested union, every sub-model + # constraint discovered here belongs to the outer arm. Only the + # outermost union picks a `field_arm` from its own variant sources, + # and only when the field is variant-specific to a single arm. + field_arm = arm if arm is not None else _singleton_arm(values) + checks, sub_model_checks = _build_field_checks( + af.field_spec, prefix=prefix, arm=field_arm + ) + model_checks.extend(sub_model_checks) + if values and discriminator is not None: + _guard_struct_nested_variant_fields(prefix, spec.name) + # Outer guards land first so the renderer composes + # outer-then-inner (e.g. a `ColumnGuard` from a parent union, + # then an `ElementGuard` from the nested union the field + # lives in). + guard: Guard = guard_cls(discriminator=discriminator, values=values) + checks = [replace(ck, guards=(guard, *ck.guards)) for ck in checks] + field_checks.extend(checks) + return field_checks, model_checks + + +def _model_checks_for_union( + spec: UnionSpec, + arm_by_class: dict[type[BaseModel], str], + target: FieldPath = ScalarPath(), + *, + arm: str | None = None, +) -> list[ModelCheck]: + """Build ModelChecks for the union itself plus each member's own constraints. + + When `arm` is None (top-level union): union-level constraints carry + `arm=None` because they apply regardless of which arm matches. + Member-class constraints (e.g. `@radio_group` on `RoadSegment`) are + tagged with the discriminator value mapped to that class so the test + renderer can confine them to the right per-arm test module. + + When `arm` is set (nested union reached from an outer arm): every + check produced -- union-level and member-level -- inherits that outer + arm. The inner union's own discriminator is irrelevant to per-arm + test filtering, which always keys on the outermost union's + discriminator. + """ + model_checks = _dispatch_model_constraints( + spec.constraints, + spec.fields, + target=target, + arm=arm, + ) + for member in spec.member_specs: + member_constraints = ModelConstraint.get_model_constraints(member.member_cls) + member_arm = arm if arm is not None else arm_by_class.get(member.member_cls) + model_checks.extend( + _dispatch_model_constraints( + member_constraints, + member.spec.fields, + target=target, + arm=member_arm, + ) + ) + return model_checks + + +def _exclusivity_checks_for_union( + spec: UnionSpec, + value_by_class: dict[type[BaseModel], str], + target: FieldPath = ScalarPath(), + *, + arm: str | None = None, +) -> list[ModelCheck]: + """Generate forbid_if/require_if checks from union variant structure. + + Unlike `dispatch_model_constraint` (which maps user-declared + `ModelConstraint` objects to descriptors), this synthesizes + `ForbidIf`/`RequireIf` descriptors directly from the union's variant + grouping. The input is a structural property of the union, not a + declared constraint, so there is no source `ModelConstraint` to + dispatch from. + + `arm` is the outer-union arm threaded through when this union is + nested inside another. Inner exclusivity checks belong to that outer + arm rather than being broadcast to every arm. + """ + if spec.discriminator_mapping is None or spec.discriminator_field is None: + return [] + + all_values = set(spec.discriminator_mapping) + + grouped: dict[str, set[type[BaseModel]]] = defaultdict(set) + required_by_field: dict[str, set[type[BaseModel]]] = defaultdict(set) + shape_by_field: dict[str, FieldShape] = {} + for af in spec.annotated_fields: + if af.variant_sources is None: + continue + name = af.field_spec.name + shape_by_field[name] = af.field_spec.shape + for src in af.variant_sources: + if src in value_by_class: + grouped[name].add(src) + if af.field_spec.is_required: + required_by_field[name].add(src) + + def forbid_check(field_name: str, condition: FieldEqCondition | Not) -> ModelCheck: + return ModelCheck( + descriptor=ForbidIf( + field_names=(field_name,), + condition=condition, + field_shapes=forbid_if_field_shapes((field_name,), shape_by_field), + ), + target=target, + arm=arm, + ) + + def require_check(field_name: str, condition: FieldEqCondition | Not) -> ModelCheck: + return ModelCheck( + descriptor=RequireIf(field_names=(field_name,), condition=condition), + target=target, + arm=arm, + ) + + checks: list[ModelCheck] = [] + disc_field = spec.discriminator_field + for field_name, variant_classes in grouped.items(): + variant_values = {value_by_class[cls] for cls in variant_classes} + excluded_values = all_values - variant_values + if not excluded_values: + continue + + if len(variant_values) == 1 and len(excluded_values) > 1: + (sole_value,) = variant_values + checks.append( + forbid_check(field_name, Not(FieldEqCondition(disc_field, sole_value))) + ) + else: + for exc_val in sorted(excluded_values): + checks.append( + forbid_check(field_name, FieldEqCondition(disc_field, exc_val)) + ) + + required_classes = required_by_field[field_name] + required_values = {value_by_class[cls] for cls in required_classes} + for req_val in sorted(required_values): + checks.append( + require_check(field_name, FieldEqCondition(disc_field, req_val)) + ) + + return checks + + +def build_checks( + spec: ModelSpec, +) -> tuple[list[Check], list[ModelCheck]]: + """Build all check IR for a feature spec. + + Roots the walk at the empty `ScalarPath()` and delegates to the same + helpers used at every nested level (`_recurse_into_union` for unions, + `_recurse_into_model` for models), so the row-root and nested cases + share one path. + """ + if isinstance(spec, UnionSpec): + return _recurse_into_union(spec) + return _recurse_into_model(spec) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_ir.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_ir.py new file mode 100644 index 000000000..b418c37c2 --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/check_ir.py @@ -0,0 +1,219 @@ +"""Tree-shaped IR for PySpark check expressions. + +Sum types describe each check's structural placement: + +- `Check.target: FieldPath` -- a `ScalarPath` or `ArrayPath` locating + where the descriptor's expression is evaluated. The choice of variant + signals whether the renderer wraps the expression in `array_check` / + `nested_array_check`. +- `Guard` -- a single discriminator gate. `Check.guards` is a tuple + of `Guard`s AND-composed at render time; nested-union gating + composes one `ColumnGuard` with one `ElementGuard`. + +The check_builder produces these types and the renderer consumes them. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TypeAlias + +from overture.schema.system.field_path import ( + ArrayPath, + FieldPath, + MapPath, + ScalarPath, + StructSegment, +) + +from .constraint_dispatch import ( + ExpressionDescriptor, + ForbidIf, + MinFieldsSet, + ModelConstraintDescriptor, + RadioGroup, + RequireAnyOf, + RequireAnyTrue, + RequireIf, + require_field_eq, +) + +__all__ = [ + "Check", + "ColumnGuard", + "ElementGuard", + "Guard", + "ModelCheck", +] + + +@dataclass(frozen=True, slots=True) +class ColumnGuard: + """Discriminator gate where the discriminator is a top-level row column.""" + + discriminator: str + values: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class ElementGuard: + """Discriminator gate where the discriminator is a struct field inside an array element.""" + + discriminator: str + values: tuple[str, ...] + + +Guard: TypeAlias = ColumnGuard | ElementGuard + + +def _top_level(name: str) -> str: + """Strip a dotted field name to its top-level column.""" + return name.split(".", 1)[0] + + +def _path_top_column(path: FieldPath) -> str | None: + """Top-level row column for a `FieldPath`, or `None` for an empty `ScalarPath`. + + Collapses dotted struct navigation to its first segment -- the granularity + at which `validate_model` detects column absence. `ArrayPath.column_path` + and `MapPath.map_column` may be dotted when the iterated column is nested + inside a struct (e.g. `names.rules`); this strips to `names`. + """ + match path: + case ScalarPath(segments=(StructSegment(name=first), *_)): + return first + case ScalarPath(): + return None + case ArrayPath(): + return _top_level(path.column_path) + case MapPath(): + return _top_level(path.map_column) + case _: + raise TypeError(f"Unhandled FieldPath variant: {type(path).__name__}") + + +@dataclass(frozen=True, slots=True) +class Check: + """A field-level validation check.""" + + descriptors: tuple[ExpressionDescriptor, ...] + target: FieldPath + guards: tuple[Guard, ...] = () + + @property + def read_columns(self) -> frozenset[str]: + """Top-level row columns this check's expression dereferences. + + Includes the target's outermost column, any `ColumnGuard` discriminator + (rendered as `F.col(...)`), and any descriptor gate on a `ScalarPath` + target (rendered as `F.col("{gate}").isNotNull()`). `ElementGuard` + discriminators are excluded -- they reference `el[...]`, an + element-relative accessor, not a row-level column. Descriptor gates on + `ArrayPath` targets are also excluded -- they are applied element-relatively + via `element_relative_gate`. + """ + cols: set[str] = set() + top = _path_top_column(self.target) + if top is not None: + cols.add(top) + for guard in self.guards: + match guard: + case ColumnGuard(discriminator=d): + cols.add(d) + case ElementGuard(): + pass # element-relative: not a row-level read + case _: + raise TypeError(f"Unhandled Guard variant: {type(guard).__name__}") + if isinstance(self.target, ScalarPath): + for desc in self.descriptors: + if desc.gate is not None: + gate_col = _path_top_column(desc.gate) + if gate_col is not None: + cols.add(gate_col) + return frozenset(cols) + + +@dataclass(frozen=True, slots=True) +class ModelCheck: + """A model-level validation check (cross-field constraint). + + `target` locates the model the constraint applies to: an empty + `ScalarPath()` for row-root constraints, or an `ArrayPath` when the + constrained model is reached by iterating one or more arrays. The + default `ScalarPath()` makes the row-root case ergonomic at + construction sites and is the common case; `Check.target` has no + sensible default and is required. + + `arm` records the discriminator value of the union member that + contributed the constraint, or `None` when the constraint applies to + every arm. The test renderer filters per-arm test modules by this + value. Constraints discovered through a variant-specific field's + sub-model or sub-union inherit the contributing outer arm, so they + land only in that arm's test module. + + `gate` is the optional-ancestor path that must be non-null for the + constraint to apply. Set when the constrained model is reached via + an optional field (`field: Model | None`). The renderer wraps the + constraint expression in `F.when(.isNotNull(), ...)` so + the check is skipped when the optional model is absent (NULL). + `gate` is always applied element-relatively for array targets and + must be `None` for scalar targets, so it never contributes a + top-level row column to `read_columns`. + """ + + descriptor: ModelConstraintDescriptor + target: FieldPath = ScalarPath() + arm: str | None = None + gate: FieldPath | None = None + + @property + def read_columns(self) -> frozenset[str]: + """Top-level row columns this model check's expression dereferences. + + For row-root constraints (`ScalarPath` target): all `field_names` from + the constraint (collapsed to top-level column) and, for `RequireIf`/ + `ForbidIf`, the condition field (both rendered as `F.col(...)`). + + For array/map targets: only the outermost container column is a + row-level read (`array_check("col", ...)` / `map_values_check("col", + ...)`). The `field_names` and condition field are accessed as + element-relative `el[...]` / `inner[...]` accessors inside the + lambda -- not as `F.col(...)` -- so they do not contribute top-level + column reads. + + `gate` is excluded: for array targets it is element-relative; for scalar + targets the renderer asserts it is `None`. The `arm` field carries no + column information. + """ + cols: set[str] = set() + desc = self.descriptor + # Array/map targets wrap everything in array_check/map_values_check; + # field references inside the lambda are element-relative, not row-level. + # Only the container column itself is a top-level read. + if not isinstance(self.target, ScalarPath): + container_col = _path_top_column(self.target) + if container_col is not None: + cols.add(container_col) + return frozenset(cols) + # Row-root target: field_names and condition field render as F.col(...). + match desc: + case ( + RequireAnyOf(field_names=names) + | RadioGroup(field_names=names) + | RequireAnyTrue(field_names=names) + | MinFieldsSet(field_names=names) + ): + for name in names: + cols.add(_top_level(name)) + case ( + RequireIf(field_names=names, condition=cond) + | ForbidIf(field_names=names, condition=cond) + ): + for name in names: + cols.add(_top_level(name)) + cols.add(require_field_eq(cond).field_name) + case _: + raise TypeError( + f"Unhandled ModelConstraintDescriptor variant: {type(desc).__name__}" + ) + return frozenset(cols) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/constraint_dispatch.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/constraint_dispatch.py new file mode 100644 index 000000000..be3008303 --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/constraint_dispatch.py @@ -0,0 +1,741 @@ +"""Constraint type to PySpark expression descriptor dispatch. + +Pure mapping from constraint objects to expression descriptors. +No awareness of field paths, list depth, or struct nesting -- +those are composition concerns handled by check_builder. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any, NamedTuple, TypeAlias + +from annotated_types import Ge, Gt, Interval, Le, Lt +from pydantic import Strict +from pydantic._internal._fields import PydanticMetadata + +from overture.schema.system.case import to_snake_case +from overture.schema.system.field_constraint.collection import UniqueItemsConstraint +from overture.schema.system.field_constraint.string import ( + JsonPointerConstraint, + PatternConstraint, + StrippedConstraint, +) +from overture.schema.system.field_path import FieldPath +from overture.schema.system.model_constraint import ( + Condition, + FieldEqCondition, + ForbidIfConstraint, + MinFieldsSetConstraint, + NoExtraFieldsConstraint, + Not, + RadioGroupConstraint, + RequireAnyOfConstraint, + RequireAnyTrueConstraint, + RequireIfConstraint, +) +from overture.schema.system.primitive import GeometryTypeConstraint +from overture.schema.system.ref import Reference + +from ..extraction.docstring import first_docstring_line +from ..extraction.field import FieldShape, ModelRef, Primitive +from ..extraction.field_walk import has_array_layer, terminal_of +from ..extraction.length_constraints import ( + ArrayMaxLen, + ArrayMinLen, + ScalarMaxLen, + ScalarMinLen, +) +from ..extraction.literal_alternatives import LiteralAlternatives +from ..extraction.specs import FieldSpec +from ..extraction.type_registry import primitive_spark_category +from ._primitive_fill import PRIMITIVE_FILL_TABLE + +__all__ = [ + "ExpressionDescriptor", + "FieldEq", + "ForbidIf", + "MinFieldsSet", + "ModelConstraintDescriptor", + "RadioGroup", + "RequireAnyOf", + "RequireAnyTrue", + "RequireIf", + "dispatch_base_type", + "dispatch_constraint", + "dispatch_model_constraint", + "dispatch_newtype", + "forbid_if_field_shapes", + "model_constraint_function", + "model_mutation_function", + "parse_field_eq", + "require_bool_field_eq", + "require_field_eq", +] + + +@dataclass(frozen=True, slots=True) +class ExpressionDescriptor: + """Describes a constraint_expressions function call. + + `function` names the function (e.g., `"check_bounds"`). + `args` are positional arguments after `col` and `field`. + `kwargs` are keyword arguments, stored as a tuple of `(name, value)` + pairs so the descriptor is hashable -- consumers convert with `dict()` + when they need mapping access. + `constraint_type` is the Python class of the constraint that + produced this descriptor (e.g., `NoWhitespaceConstraint`), + used by test generators to pick pattern-appropriate mutation values. + `gate` is the structural path to a nullable ancestor struct; when set, + the renderer wraps the expression in `F.when(gate.isNotNull(), ...)`. + `label` is a human-readable description used in error messages + (e.g., `"ISO 3166-1 alpha-2 country code"`). + `check_name` overrides the Check.name derivation in error_key; + when None, the renderer strips the `check_` prefix from `function`. + """ + + function: str + args: tuple[object, ...] = () + kwargs: tuple[tuple[str, object], ...] = () + constraint_type: type | None = None + gate: FieldPath | None = None + label: str | None = None + check_name: str | None = None + check_nan: bool | None = None + allow_literals: tuple[object, ...] = () + """Literal values that bypass this check. + + When non-empty, the renderer wraps the generated call in + `except_literals(col, call, list(allow_literals))` so that a column + value matching one of these literals is treated as valid regardless of + what the check would otherwise report. Populated by `check_builder` + from `LiteralAlternatives` constraints on terminal scalars (e.g. + `HttpUrl | Literal[""]`). Never set on `check_required` descriptors. + """ + + +_BASE_TYPE_DISPATCH: dict[str, tuple[ExpressionDescriptor, ...]] = { + "HttpUrl": ( + ExpressionDescriptor(function="check_url_format"), + ExpressionDescriptor(function="check_url_length"), + ), + "EmailStr": (ExpressionDescriptor(function="check_email"),), + "BBox": ( + ExpressionDescriptor(function="check_bbox_completeness"), + ExpressionDescriptor(function="check_bbox_lat_ordering"), + ExpressionDescriptor(function="check_bbox_lat_range"), + ), +} + +_NEWTYPE_DISPATCH: dict[str, tuple[ExpressionDescriptor, ...]] = { + "LinearlyReferencedRange": ( + ExpressionDescriptor(function="check_linear_range_length"), + ExpressionDescriptor(function="check_linear_range_bounds"), + ExpressionDescriptor(function="check_linear_range_order"), + ), +} + + +# re.UNICODE is Python's implicit default on compiled `str` patterns and needs +# no translation -- Java's regex engine is Unicode-aware without a flag. +# re.IGNORECASE maps to the inline `(?i)` flag Spark's rlike honors. A new +# supported flag with a visible matching effect also belongs in +# `field_constraints._DISPLAY_FLAG_LETTERS`, or docs will hide its behavior. +_SUPPORTED_PATTERN_FLAGS = re.IGNORECASE | re.UNICODE + + +def compiled_pattern_source(pattern: re.Pattern[str]) -> str: + """Return the Spark-regex source string for a compiled `re.Pattern`. + + A compiled `re.Pattern` is the only Pydantic carrier for a flagged pattern + (a bare `Field(pattern=str)` cannot express `re.I`). Translates the flags + Spark's `rlike` can honor into inline prefixes -- `re.IGNORECASE` becomes + `(?i)`, the idiom `constraint_expressions.check_url_format` already uses. + The ASCII/Unicode case-folding divergence between Java and Python is the + same accepted divergence documented at `check_pattern`. + + Raises + ------ + NotImplementedError + For any flag without a faithful `rlike` translation (e.g. + `re.MULTILINE`), naming the flag rather than silently dropping it. + """ + unsupported = re.RegexFlag(pattern.flags & ~_SUPPORTED_PATTERN_FLAGS) + if unsupported: + raise NotImplementedError( + f"check_pattern cannot translate regex flag {unsupported!r} to Spark rlike" + ) + source = pattern.pattern + # Only IGNORECASE emits a prefix; UNICODE passes the gate but is a no-op + # (Java is Unicode-aware unflagged). A new supported flag needs its own + # translation clause here, or it will pass the gate and be silently dropped. + if pattern.flags & re.IGNORECASE: + source = f"(?i){source}" + return source + + +def normalize_anchor(pattern: str) -> str: + """Replace trailing `$` with `\\z` for Java/Spark regex compatibility. + + Uses backslash-parity to distinguish a real anchor from an escaped + literal `$`. Counts the run of backslashes immediately before the + final `$`: an even count means `$` is unescaped (convert to `\\z`); + an odd count means it is an escaped literal `$` (leave unchanged). + """ + if not pattern.endswith("$"): + return pattern + prefix = pattern[:-1] # strip the trailing $ + backslashes = len(prefix) - len(prefix.rstrip("\\")) + if backslashes % 2 == 0: + return prefix + r"\z" + return pattern + + +def _pattern_check_name(constraint: PatternConstraint) -> str: + """Derive a snake_case check name from the constraint class name.""" + if type(constraint) is PatternConstraint: + return "pattern" + return to_snake_case(type(constraint).__name__.removesuffix("Constraint")) + + +def _pattern_label(constraint: PatternConstraint) -> str: + """Extract a human-readable label from a PatternConstraint.""" + if constraint.description: + return constraint.description + if (summary := first_docstring_line(type(constraint).__doc__)) is not None: + return summary.rstrip(".") + name = type(constraint).__name__.removesuffix("Constraint") + return to_snake_case(name).replace("_", " ") + + +_ConstraintHandler = Callable[[Any, str | None], ExpressionDescriptor | None] + + +_BOUND_ATTRS = ("ge", "gt", "le", "lt") + + +def _dispatch_bounds( + constraint: Ge | Gt | Le | Lt | Interval, + base_type: str | None, +) -> ExpressionDescriptor: + """Extract bound kwargs from an annotated_types constraint. + + Coerces integer bound values to float on float-typed columns so + that generated test mutations match the Spark DoubleType column. + """ + is_float = base_type is not None and primitive_spark_category(base_type) == "float" + kwargs: list[tuple[str, object]] = [] + for attr in _BOUND_ATTRS: + value = getattr(constraint, attr, None) + if value is not None: + if is_float and isinstance(value, int) and not isinstance(value, bool): + value = float(value) + kwargs.append((attr, value)) + check_nan: bool | None = False if base_type is not None and not is_float else None + return ExpressionDescriptor( + function="check_bounds", kwargs=tuple(kwargs), check_nan=check_nan + ) + + +def _dispatch_pattern( + constraint: PatternConstraint, + _base_type: str | None, +) -> ExpressionDescriptor: + """Map a PatternConstraint (or subclass) to a check_pattern descriptor. + + The Python `re` pattern source is embedded verbatim (anchor and inline + flags aside) into a Java `rlike`. The two engines diverge on Unicode + shorthand classes and `.` line-terminator handling; that is an accepted + divergence, documented at `constraint_expressions.check_pattern`. + """ + return ExpressionDescriptor( + function="check_pattern", + args=(normalize_anchor(compiled_pattern_source(constraint.pattern)),), + constraint_type=type(constraint), + label=_pattern_label(constraint), + check_name=_pattern_check_name(constraint), + ) + + +def _raw_pattern(constraint: object) -> str | None: + """Return the Spark-regex source of raw pydantic `Field(pattern=)`, or None. + + Pydantic represents `Field(pattern=...)` as a `PydanticMetadata` marker + (the private `_PydanticGeneralMetadata`) carrying the pattern as either a + `str` (`Field(pattern="...")`) or a compiled `re.Pattern` + (`Field(pattern=re.compile(...))` -- the only carrier for a flagged, + e.g. case-insensitive, pattern). The schema's own `PatternConstraint` is + handled earlier; raw metadata reaches here from `dict[K, V]` keys/values + that used `Field(pattern=)` rather than a schema constraint class + (e.g. `Sources.license_priority`). + + The `PydanticMetadata` check -- not merely a `.pattern` attribute -- + keeps `dispatch_constraint`'s fallback contract intact: an unrelated future + constraint that happens to expose a `.pattern` still raises `TypeError` + rather than being silently dispatched as a `check_pattern`. A compiled + pattern carrying an untranslatable flag raises `NotImplementedError` via + `compiled_pattern_source`. + """ + if not isinstance(constraint, PydanticMetadata): + return None + pattern = getattr(constraint, "pattern", None) + if isinstance(pattern, str): + return pattern + if isinstance(pattern, re.Pattern): + return compiled_pattern_source(pattern) + return None + + +# Ordered: the first matching entry wins, so any subclass relationship +# between keys must place the more-specific class first. StrippedConstraint +# subclasses PatternConstraint, so it must appear before the PatternConstraint +# fallback entry. +_CONSTRAINT_DISPATCH: list[tuple[type | tuple[type, ...], _ConstraintHandler]] = [ + # LiteralAlternatives is a modifier threaded onto the field's other + # descriptors as allow_literals (by check_builder), not a standalone check. + ((Reference, Strict, LiteralAlternatives), lambda _c, _bt: None), + ((Ge, Gt, Le, Lt, Interval), _dispatch_bounds), + ( + ArrayMinLen, + lambda c, _bt: ExpressionDescriptor( + function="check_array_min_length", args=(c.min_length,) + ), + ), + ( + ArrayMaxLen, + lambda c, _bt: ExpressionDescriptor( + function="check_array_max_length", args=(c.max_length,) + ), + ), + ( + ScalarMinLen, + lambda c, _bt: ExpressionDescriptor( + function="check_string_min_length", args=(c.min_length,) + ), + ), + ( + ScalarMaxLen, + lambda c, _bt: ExpressionDescriptor( + function="check_string_max_length", args=(c.max_length,) + ), + ), + ( + StrippedConstraint, + lambda c, _bt: ExpressionDescriptor( + function="check_stripped", constraint_type=type(c) + ), + ), + ( + JsonPointerConstraint, + lambda c, _bt: ExpressionDescriptor( + function="check_json_pointer", constraint_type=type(c) + ), + ), + (PatternConstraint, _dispatch_pattern), + # check_struct_unique uses Spark's array_distinct: structural equality on + # whole elements, against the raw stored values. Pydantic's + # UniqueItemsConstraint on list[HttpUrl] compares *normalized* URLs + # (trailing-slash, lowercase host/scheme), so it catches duplicates that + # differ only in normalization. We accept that difference -- the PySpark + # check catches exact duplicates only. + ( + UniqueItemsConstraint, + lambda _c, _bt: ExpressionDescriptor(function="check_struct_unique"), + ), + ( + GeometryTypeConstraint, + lambda c, _bt: ExpressionDescriptor( + function="check_geometry_type", args=tuple(c.allowed_types) + ), + ), +] + + +def dispatch_constraint( + constraint: object, + *, + base_type: str | None = None, +) -> ExpressionDescriptor | None: + """Map a constraint object to an expression descriptor. + + Parameters + ---------- + constraint + The constraint object from `ConstraintSource.constraint`. Length + constraints arrive as `ArrayMinLen` / `ArrayMaxLen` / + `ScalarMinLen` / `ScalarMaxLen` -- the typed variants emitted + by `extraction.type_analyzer.attach_constraints`. + base_type + The field's terminal-scalar base type, used to detect float + bounds. + + Returns + ------- + ExpressionDescriptor or None + `None` for explicitly skipped constraints (Reference, Strict). + + Raises + ------ + TypeError + For unrecognized constraint types. + """ + for key_types, handler in _CONSTRAINT_DISPATCH: + if isinstance(constraint, key_types): + return handler(constraint, base_type) + raw_pattern = _raw_pattern(constraint) + if raw_pattern is not None: + # Raw pydantic `Field(pattern=)` metadata. `constraint_type` stays + # None (the pydantic class is a private closure type, not a stable + # key); the curated valid/invalid pair lives in `PATTERN_VALUES`, + # keyed by the normalized pattern in `args`. + return ExpressionDescriptor( + function="check_pattern", + args=(normalize_anchor(raw_pattern),), + label="pattern", + ) + raise TypeError(f"Unhandled constraint type: {type(constraint).__name__}") + + +def dispatch_newtype(newtype_name: str) -> tuple[ExpressionDescriptor, ...] | None: + """Look up a NewType-level expression override. + + Returns None when the NewType decomposes normally into + individual constraint dispatches. + """ + return _NEWTYPE_DISPATCH.get(newtype_name) + + +def dispatch_base_type(base_type: str) -> tuple[ExpressionDescriptor, ...] | None: + """Look up a base-type-level expression override. + + Handles primitive types like HttpUrl and EmailStr that carry no + Annotated constraints but need semantic validation functions. + """ + return _BASE_TYPE_DISPATCH.get(base_type) + + +class FieldEq(NamedTuple): + """An unwrapped `FieldEqCondition`, with `negated` set when wrapped in `Not`.""" + + field_name: str + value: object + negated: bool + + +def parse_field_eq(condition: Condition) -> FieldEq | None: + """Unwrap a `FieldEqCondition` or `Not(FieldEqCondition)`. + + Returns a `FieldEq` triple for either shape, or `None` for any + other condition. `negated` is True iff the condition was wrapped + in `Not`. + """ + match condition: + case Not(inner=FieldEqCondition(field_name=fn, value=v)): + return FieldEq(fn, v, True) + case FieldEqCondition(field_name=fn, value=v): + return FieldEq(fn, v, False) + case _: + return None + + +def require_field_eq(condition: Condition) -> FieldEq: + """Unwrap a field-equality condition, raising on any other shape. + + The strict companion to `parse_field_eq`, for callers that only + handle `FieldEqCondition` / `Not(FieldEqCondition)`: a new condition + subtype fails loudly here, in one place, rather than slipping through + several independent `None` checks with drifting error messages. + """ + parsed = parse_field_eq(condition) + if parsed is None: + raise TypeError(f"Unhandled condition type: {type(condition).__name__}") + return parsed + + +def require_bool_field_eq(condition: Condition) -> FieldEq: + """Unwrap a positive boolean `FieldEqCondition`, raising otherwise. + + `require_any_true` PySpark support is limited to positive boolean-flag + equalities: the runtime coalesces a null condition to False (sound only + for a positive equality), and the test-data disabling value is the + boolean's negation. Negation or a non-boolean value raises here, so + every consumer -- dispatch, base-row synthesis, test rendering -- accepts + exactly the same set rather than each enforcing a different subset. + """ + field_eq = require_field_eq(condition) + if field_eq.negated or not isinstance(field_eq.value, bool): + raise TypeError( + "require_any_true PySpark generation supports only positive " + f"boolean FieldEqConditions; got {condition!r}" + ) + return field_eq + + +@dataclass(frozen=True, slots=True) +class RequireAnyOf: + """Descriptor for `check_require_any_of`: at least one field must be set.""" + + field_names: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class RequireAnyTrue: + """Descriptor for `check_require_any_true`: at least one condition holds. + + Carries the raw `Condition`s so the renderer can lower each to a + Spark boolean expression (reusing the `require_if` condition path). + Unlike `RequireAnyOf`, which tests field presence, each condition + tests a field value -- the divisions case is all + `FieldEqCondition(field, True)`. + + Construction enforces the positive-boolean invariant (via + `require_bool_field_eq`), so every instance is valid by construction and + the `field_names` and renderer paths need not re-check. + """ + + conditions: tuple[Condition, ...] + + def __post_init__(self) -> None: + for c in self.conditions: + require_bool_field_eq(c) + + @property + def field_names(self) -> tuple[str, ...]: + """Fields the conditions reference -- the columns this constraint governs. + + Derived from `conditions` so `RequireAnyTrue` presents the same + `field_names` interface as the presence-based descriptors, letting + generic consumers (read-column tracking, node filtering) treat every + model-constraint descriptor uniformly. The value-vs-presence + distinction is carried by the descriptor type, not this attribute. + """ + return tuple(require_field_eq(c).field_name for c in self.conditions) + + +@dataclass(frozen=True, slots=True) +class RadioGroup: + """Descriptor for `check_radio_group`: exactly one boolean field must be True.""" + + field_names: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class RequireIf: + """Descriptor for `check_require_if`: field required when condition holds.""" + + field_names: tuple[str, ...] + condition: Condition + + +@dataclass(frozen=True, slots=True) +class ForbidIf: + """Descriptor for `check_forbid_if`: field must be absent when condition holds. + + `field_shapes` pairs non-string-default field names with their `FieldShape` + so the test renderer can emit type-appropriate `fill_values` literals. + Stored as a tuple of `(name, shape)` pairs so the descriptor is + hashable; consumers convert with `dict()` when they need mapping + access. String fields are omitted because the renderer defaults to + `""` for them, which is correct. Arrays, model references, and + non-string scalars (int/uint/float/bool) require an explicit entry + so the renderer emits a typed literal (`[{}]`, `{}`, `0`, `False`, etc.). + """ + + field_names: tuple[str, ...] + condition: Condition + field_shapes: tuple[tuple[str, FieldShape], ...] + + +@dataclass(frozen=True, slots=True) +class MinFieldsSet: + """Descriptor for `check_min_fields_set`: at least `count` fields set. + + Matches Pydantic's `model_fields_set` semantics: required fields are + always set (the constructor requires them) and contribute to the count + alongside any explicitly-set optional fields. Both kinds are passed to + the runtime check. + """ + + field_names: tuple[str, ...] + count: int + + +ModelConstraintDescriptor: TypeAlias = ( + RequireAnyOf | RequireAnyTrue | RadioGroup | RequireIf | ForbidIf | MinFieldsSet +) +"""One variant per model-constraint kind. + +Each variant carries only the fields meaningful for that constraint; +`ForbidIf` adds `field_shapes` for non-string targets so the test +renderer can emit type-appropriate `fill_values` literals. +""" + + +def _first_required_leaf(field_spec: FieldSpec) -> str | None: + """Return the name of the first required field in a MODEL-kind `FieldSpec`. + + Returns `None` for fields whose terminal is anything but a + `ModelRef` (scalars, arrays, `UnionRef`s, etc.). The + `RequireAnyOf` unwrapping uses this to drill into a struct's + required leaf when one exists; non-struct terminals leave the + field name unwrapped, which is the correct behavior for scalars + and arrays. `UnionRef` returns `None` because picking one arm's + required leaf would silently bias the constraint to that arm. + """ + if has_array_layer(field_spec.shape): + return None + terminal = terminal_of(field_spec.shape) + if not isinstance(terminal, ModelRef): + return None + for sub in terminal.model.fields: + if sub.is_required: + return sub.name + return None + + +def _unwrap_require_any_of_names( + field_names: tuple[str, ...], + by_name: dict[str, FieldSpec], +) -> tuple[str, ...]: + """Replace struct field names with their first required leaf path.""" + result = [] + for name in field_names: + field_spec = by_name.get(name) + leaf = _first_required_leaf(field_spec) if field_spec is not None else None + result.append(f"{name}.{leaf}" if leaf is not None else name) + return tuple(result) + + +def _needs_explicit_fill(shape: FieldShape) -> bool: + """Whether `shape` needs an explicit (non-default-string) fill value. + + Arrays and model references need `[{}]` / `{}` fill. Non-string + scalars (int/uint/float/bool families) need a typed fill (0, False, + etc.). Plain string scalars are omitted -- the `""` default is correct. + """ + if has_array_layer(shape): + return True + terminal = terminal_of(shape) + if isinstance(terminal, ModelRef): + return True + if not isinstance(terminal, Primitive): + return False + return primitive_spark_category(terminal.base_type) in PRIMITIVE_FILL_TABLE + + +def forbid_if_field_shapes( + field_names: tuple[str, ...], + shape_by_name: Mapping[str, FieldShape], +) -> tuple[tuple[str, FieldShape], ...]: + """Build the `field_shapes` pairs for non-string ForbidIf targets. + + Keeps fields whose shape is an array, a model reference, or a + non-string scalar (int/uint/float/bool families). String fields are + omitted because the test renderer defaults their fill value to `""` + without needing the shape. + """ + return tuple( + (name, shape) + for name in field_names + if (shape := shape_by_name.get(name)) is not None + and _needs_explicit_fill(shape) + ) + + +def dispatch_model_constraint( + constraint: object, + fields: list[FieldSpec], +) -> tuple[ModelConstraintDescriptor, ...]: + """Map a model-level constraint to fully constructed typed descriptors. + + Parameters + ---------- + constraint + The model constraint object. + fields + All fields of the model. Branches consult them as needed -- + `RequireAnyOf` and `ForbidIf` index by name, `MinFieldsSet` + enumerates every field (required and optional). + + Returns + ------- + tuple of ModelConstraintDescriptor + Empty tuple for explicitly skipped constraints (NoExtraFields). + Most kinds return a single-element tuple. Multi-field + `@require_if` / `@forbid_if` split into one descriptor per + target field because the runtime check functions take a single + target column each. + + Raises + ------ + TypeError + For unrecognized constraint types. + """ + match constraint: + case NoExtraFieldsConstraint(): + return () + case RequireAnyOfConstraint(): + unwrapped = _unwrap_require_any_of_names( + constraint.field_names, {f.name: f for f in fields} + ) + return (RequireAnyOf(field_names=unwrapped),) + case RequireAnyTrueConstraint(): + # `RequireAnyTrue.__post_init__` enforces the positive-boolean set + # the whole pipeline (runtime, renderer, base row, test renderer) + # depends on, so a bad condition raises here at construction. + return (RequireAnyTrue(conditions=constraint.conditions),) + case RadioGroupConstraint(): + return (RadioGroup(field_names=constraint.field_names),) + case RequireIfConstraint(): + # `@require_if(["a", "b"], cond)` means "all of a, b required when + # cond" -- one runtime check per field, since check_require_if + # takes a single target column. + return tuple( + RequireIf(field_names=(name,), condition=constraint.condition) + for name in constraint.field_names + ) + case ForbidIfConstraint(): + shapes_by_field = forbid_if_field_shapes( + constraint.field_names, + {f.name: f.shape for f in fields}, + ) + per_field_shapes = dict(shapes_by_field) + return tuple( + ForbidIf( + field_names=(name,), + condition=constraint.condition, + field_shapes=( + ((name, per_field_shapes[name]),) + if name in per_field_shapes + else () + ), + ) + for name in constraint.field_names + ) + case MinFieldsSetConstraint(): + all_names = tuple(f.name for f in fields) + return (MinFieldsSet(field_names=all_names, count=constraint.count),) + case _: + raise TypeError(f"Unhandled model constraint: {type(constraint).__name__}") + + +_MODEL_CONSTRAINT_DISPATCH: dict[type[ModelConstraintDescriptor], tuple[str, str]] = { + RequireAnyOf: ("check_require_any_of", "mutate_require_any_of"), + RequireAnyTrue: ("check_require_any_true", "mutate_require_any_true"), + RadioGroup: ("check_radio_group", "mutate_radio_group"), + RequireIf: ("check_require_if", "mutate_require_if"), + ForbidIf: ("check_forbid_if", "mutate_forbid_if"), + MinFieldsSet: ("check_min_fields_set", "mutate_min_fields_set"), +} + + +def model_constraint_function(d: ModelConstraintDescriptor) -> str: + """Map a `ModelConstraintDescriptor` variant to its runtime function name.""" + return _MODEL_CONSTRAINT_DISPATCH[type(d)][0] + + +def model_mutation_function(d: ModelConstraintDescriptor) -> str: + """Map a `ModelConstraintDescriptor` variant to its test mutation helper.""" + return _MODEL_CONSTRAINT_DISPATCH[type(d)][1] diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/pipeline.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/pipeline.py new file mode 100644 index 000000000..9bb2a9daa --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/pipeline.py @@ -0,0 +1,256 @@ +"""PySpark generation pipeline: produce modules without I/O. + +Orchestrates check building, schema building, and rendering into +GeneratedModule objects. The caller decides what to do with them (write +to disk, stream to stdout, etc.). +""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from pathlib import PurePosixPath + +from overture.schema.system.case import to_snake_case +from overture.schema.system.discovery import entry_point_to_path +from overture.schema.system.primitive import GeometryType + +from ..extraction.specs import ModelSpec, UnionSpec +from .check_builder import build_checks +from .check_ir import Check, ModelCheck +from .renderer import render_model_module +from .schema_builder import build_schema +from .test_data.base_row import ( + generate_arm_rows, + generate_base_row, + generate_populated_arm_rows, + generate_populated_row, +) +from .test_renderer import render_test_module + +__all__ = [ + "GeneratedModule", + "PipelineOutput", + "generate_pyspark_module", + "generate_pyspark_modules", +] + + +@dataclass(frozen=True, slots=True) +class GeneratedModule: + """A generated Python module with its content and output path.""" + + content: str + path: PurePosixPath + + +@dataclass(frozen=True, slots=True) +class PipelineOutput: + """PySpark modules emitted by the pipeline, split by output tree. + + The source and test trees are written to separate directories and + mirror the same relative layout, so a path is meaningful only + relative to its tree. Splitting at the boundary keeps each tree + self-contained -- in practice the overlap today is just + `__init__.py`, but any path duplicated between trees would be + ambiguous in a single flat list. + """ + + source: list[GeneratedModule] + test: list[GeneratedModule] + + +_OUTPUT_PACKAGE = "overture.schema.pyspark.expressions.generated" + +# Dots in `from ...x import y` from a generated test module to reach +# `tests/`: one to leave the file's package, one to leave `generated/`. +# Each additional directory component under `generated/` adds another. +_DOTS_FROM_TEST_TO_TESTS_ROOT = 2 + + +def _support_prefix(directory: PurePosixPath) -> str: + """Relative-import prefix used by generated test modules to reach `_support`. + + Each leading dot climbs one package level; the first two dots step + out of `tests/generated/` to `tests/`, and an extra dot is appended + for every component of *directory* under `generated/`. + """ + return "." * (len(directory.parts) + _DOTS_FROM_TEST_TO_TESTS_ROOT) + + +def _require_entry_point(spec: ModelSpec) -> str: + """Return *spec*'s entry point or raise if it's missing.""" + if spec.entry_point is None: + msg = f"ModelSpec {spec.name!r} has no entry_point." + raise ValueError(msg) + return spec.entry_point + + +def _directory_and_model_name(spec: ModelSpec) -> tuple[PurePosixPath, str]: + """Return the output directory and snake_case model name for a spec. + + Both halves derive from the entry-point's class name so filenames + and symbol names stay in sync with what the runtime registry + discovers. + """ + directory, cls_name = entry_point_to_path(_require_entry_point(spec)) + return directory, to_snake_case(cls_name) + + +def _extract_geometry_types( + field_checks: list[Check], +) -> tuple[GeometryType, ...]: + """Collect allowed geometry types from every `check_geometry_type` descriptor. + + A model may carry multiple `check_geometry_type` descriptors -- e.g. + one per union arm with a distinct allowed-types set. The result is the + union of all of them, sorted by name for deterministic output. + """ + seen: set[GeometryType] = set() + for check in field_checks: + for desc in check.descriptors: + if desc.function != "check_geometry_type": + continue + for arg in desc.args: + if isinstance(arg, GeometryType): + seen.add(arg) + return tuple(sorted(seen, key=lambda g: g.name)) + + +def _init_modules(paths: Iterable[PurePosixPath]) -> list[GeneratedModule]: + """Emit empty `__init__.py` for every directory of `paths`. + + Includes the output root so the top-level package exists after a + full `rm -rf` of the generated tree. + """ + paths = list(paths) + if not paths: + return [] + dirs: set[PurePosixPath] = set() + for path in paths: + dirs.update(path.parents) + return [GeneratedModule(content="", path=d / "__init__.py") for d in sorted(dirs)] + + +def generate_pyspark_module(spec: ModelSpec) -> GeneratedModule: + """Generate a PySpark validation module from a model spec. + + Parameters + ---------- + spec + The extracted model spec to generate from. + + Returns + ------- + GeneratedModule + Module content and a relative output path mirroring the + model's entry-point package layout. + """ + return _render_module(spec, build_checks(spec)) + + +def generate_pyspark_modules( + model_specs: Sequence[ModelSpec], +) -> PipelineOutput: + """Generate PySpark validation modules for all models. + + Parameters + ---------- + model_specs + Extracted model specs to generate from. + + Returns + ------- + PipelineOutput + Source-tree model modules and test-tree modules. Each tree + includes the `__init__.py` files needed for its package layout. + """ + items = [(spec, build_checks(spec)) for spec in model_specs] + source = [_render_module(spec, checks) for spec, checks in items] + test: list[GeneratedModule] = [] + for spec, checks in items: + test.extend(_render_test_modules(spec, checks)) + source.extend(_init_modules(m.path for m in source)) + test.extend(_init_modules(m.path for m in test)) + return PipelineOutput(source=source, test=test) + + +def _render_module( + spec: ModelSpec, + checks: tuple[list[Check], list[ModelCheck]], +) -> GeneratedModule: + """Build checks, schema, and render for a model spec.""" + field_checks, model_checks = checks + schema_fields = build_schema(spec) + geometry_types = _extract_geometry_types(field_checks) + directory, model_name = _directory_and_model_name(spec) + content = render_model_module( + model_name, + field_checks, + model_checks, + schema_fields, + geometry_types, + entry_point=_require_entry_point(spec), + partitions=spec.partitions, + ) + return GeneratedModule( + content=content, + path=directory / f"{model_name}.py", + ) + + +def _select_arm_rows( + spec: ModelSpec, +) -> dict[str | None, tuple[dict[str, object], dict[str, object]]]: + """Map each test module's arm key to its (sparse, populated) base rows. + + Multi-arm unions key by discriminator value (one entry per arm); other + specs use a single `None` key. Either way the caller iterates the dict + to emit one test module per entry. + """ + if isinstance(spec, UnionSpec) and spec.discriminator_field: + sparse_arm_rows = generate_arm_rows(spec) + populated_arm_rows = generate_populated_arm_rows(spec) + return { + arm: (sparse_arm_rows[arm], populated_arm_rows[arm]) + for arm in sparse_arm_rows + } + return {None: (generate_base_row(spec), generate_populated_row(spec))} + + +def _render_test_modules( + spec: ModelSpec, + checks: tuple[list[Check], list[ModelCheck]], +) -> list[GeneratedModule]: + """Render test modules for a model spec. + + For union specs with multiple discriminator arms, produces one + test module per arm. Each arm's test includes the field and + model checks tagged with that arm (or untagged), filtered by + `render_test_module`. + """ + field_checks, model_checks = checks + directory, model_name = _directory_and_model_name(spec) + expression_import = ".".join([_OUTPUT_PACKAGE, *directory.parts, model_name]) + support_prefix = _support_prefix(directory) + + modules: list[GeneratedModule] = [] + for arm, (base_row_sparse, base_row_populated) in _select_arm_rows(spec).items(): + suffix = f"_{arm}" if arm is not None else "" + modules.append( + GeneratedModule( + content=render_test_module( + model_name, + field_checks, + model_checks, + base_row_sparse=base_row_sparse, + base_row_populated=base_row_populated, + arm=arm, + spec=spec, + expression_import=expression_import, + support_prefix=support_prefix, + ), + path=directory / f"test_{model_name}{suffix}.py", + ) + ) + return modules diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/renderer.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/renderer.py new file mode 100644 index 000000000..6fb48f3e1 --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/renderer.py @@ -0,0 +1,747 @@ +"""Render Check / ModelCheck IR into complete Python modules.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from enum import Enum + +from overture.schema.system.field_path import ( + ArrayPath, + FieldPath, + MapPath, + MapProjection, + ScalarPath, +) +from overture.schema.system.primitive import GeometryType + +from ._render_common import ( + FieldCheckRow, + ModelCheckRow, + field_check_rows, + jinja_env, + map_runtime_helper, + model_check_rows, + py_literal, + sanitize_field_name, + schema_const_name, + tuple_literal, +) +from .check_ir import ( + Check, + ColumnGuard, + ElementGuard, + ModelCheck, +) +from .constraint_dispatch import ( + ExpressionDescriptor, + FieldEq, + ForbidIf, + MinFieldsSet, + RadioGroup, + RequireAnyOf, + RequireAnyTrue, + RequireIf, + model_constraint_function, + require_field_eq, +) +from .schema_builder import SHARED_TYPE_REFS, SchemaField + +__all__ = [ + "render_model_module", +] + +# Descriptor function names that resolve to helpers from the +# `column_patterns` runtime module (rather than `constraint_expressions`). +# Used to route imports to the correct module. Distinct from +# `_render_common.COLUMN_LEVEL_FUNCTIONS`, which classifies checks that +# emit one Check per field rather than per array element. +_COLUMN_PATTERN_HELPERS = frozenset( + { + "array_check", + "nested_array_check", + "map_keys_check", + "map_values_check", + "check_struct_unique", + } +) + +_SHARED_STRUCT_REFS = frozenset(SHARED_TYPE_REFS.values()) + +_SPARK_TYPES = frozenset( + { + "ArrayType", + "BinaryType", + "BooleanType", + "ByteType", + "DateType", + "DoubleType", + "FloatType", + "IntegerType", + "LongType", + "MapType", + "ShortType", + "StringType", + "StructField", + "StructType", + "TimestampType", + } +) + + +def _render_condition_desc(parsed: FieldEq) -> str: + """Render a parsed condition to a human-readable error-message description.""" + display = repr( + parsed.value.value if isinstance(parsed.value, Enum) else parsed.value + ) + op = "!=" if parsed.negated else "=" + return f"{parsed.field_name} {op} {display}" + + +def _render_condition( + parsed: FieldEq, + *, + in_array: bool = False, + struct_path: tuple[str, ...] = (), + var: str = "el", +) -> str: + """Render a parsed condition to a PySpark Column expression string. + + `struct_path` is the leaf the constrained model was reached at; the + condition field lives beside the target field on that same model, so + its reference must navigate the same leaf (e.g. `el["inner"]["subtype"]`, + not `el["subtype"]`). + """ + ref = _render_field_ref( + parsed.field_name, in_array=in_array, struct_path=struct_path, var=var + ) + op = "!=" if parsed.negated else "==" + # A bare `== True` / `== False` -- from any boolean condition, whether + # require_if/forbid_if or require_any_true -- trips ruff's E712, which would + # rewrite the comparison away; wrap the boolean in `F.lit(...)` so the Column + # comparison survives post-generation ruff intact. + value_src = ( + f"F.lit({py_literal(parsed.value)})" + if isinstance(parsed.value, bool) + else py_literal(parsed.value) + ) + return f"{ref} {op} {value_src}" + + +def _render_field_ref( + field_name: str, + *, + in_array: bool, + struct_path: tuple[str, ...] = (), + var: str = "el", +) -> str: + """Render a field reference as F.col("x"), el["x"], or el["struct"]["x"]. + + `F.col` accepts dotted names directly so the top-level form keeps + `field_name` intact. The in-array form descends a struct via + `el[...]`, which requires the dotted name to be split into segments + before applying `struct_path` and the field's own parts. + """ + if not in_array: + return f'F.col("{field_name}")' + parts = (*struct_path, *field_name.split(".")) + return _element_accessor(var, parts) + + +def _geometry_type_literal(g: GeometryType) -> str: + """Spell out `GeometryType.NAME` as valid Python source. + + `repr(g)` yields ``, which is not a valid + expression. + """ + return f"GeometryType.{g.name}" + + +# Check functions whose first positional arg is a list of allowed values. +# Descriptors store the values as a tuple for hashability; the renderer +# unwraps that one position to a list literal so the generated call matches +# the runtime signature. +_LIST_FIRST_ARG_FUNCTIONS = frozenset({"check_enum"}) + + +def _render_arg(arg: object) -> str: + """Render a descriptor arg as a valid Python expression string.""" + if isinstance(arg, GeometryType): + return _geometry_type_literal(arg) + return py_literal(arg) + + +def _render_expr_call( + desc: ExpressionDescriptor, + col_expr: str, +) -> str: + """Render a single ExpressionDescriptor call with col injected.""" + parts = [col_expr] + for idx, arg in enumerate(desc.args): + if ( + idx == 0 + and desc.function in _LIST_FIRST_ARG_FUNCTIONS + and isinstance(arg, tuple) + ): + parts.append(py_literal(list(arg))) + else: + parts.append(_render_arg(arg)) + for k, v in desc.kwargs: + parts.append(f"{k}={py_literal(v)}") + if desc.check_nan is not None: + parts.append(f"check_nan={py_literal(desc.check_nan)}") + if desc.label is not None: + parts.append(f"label={py_literal(desc.label)}") + call = f"{desc.function}({', '.join(parts)})" + if desc.allow_literals: + literals = py_literal(list(desc.allow_literals)) + return f"except_literals({col_expr}, {call}, {literals})" + return call + + +def _element_accessor(var: str, path: tuple[str, ...]) -> str: + """Build bracket-notation accessor like `el["foo"]["bar"]`.""" + return var + "".join(f'["{p}"]' for p in path) + + +def _iter_var_name(idx: int, total: int) -> str: + """Lambda variable name at iteration depth `idx` (0..total-1) of `total`. + + Single-iteration cases (`total == 1`) return `"el"` from the first + branch; the innermost frame of a nested iteration uses `"inner"`, + intermediate frames `"el2"`, `"el3"`, ... + """ + if idx == 0: + return "el" + if idx == total - 1: + return "inner" + return f"el{idx + 1}" + + +def _wrap_element_gate(body: str, var: str, gate_parts: tuple[str, ...]) -> str: + """Wrap a lambda body in F.when(var[gate].isNotNull(), ...) for nullable parent gating.""" + gate_accessor = _element_accessor(var, gate_parts) + return f"F.when({gate_accessor}.isNotNull(), {body})" + + +def _wrap_in_array_iteration( + column_path: str, + inner_struct_paths: tuple[tuple[str, ...], ...], + body: str, + *, + gate_parts: tuple[str, ...] = (), +) -> str: + """Wrap `body` in nested array_check / nested_array_check frames. + + One frame per iteration: `column_path` names the outermost array + column, `inner_struct_paths` gives the struct accessor from each + iteration's element to the next array (its length plus one is the + iteration count). `body` is the innermost lambda body. `gate_parts`, + when set, wraps the outermost lambda body in a nullable-parent + element gate. + + The recursion descends one frame per call, carrying the frame index + and its lambda variable; the innermost frame is `array_check`, every + outer frame `nested_array_check`. + """ + total = 1 + len(inner_struct_paths) + + def frame(idx: int, accessor: str, var: str) -> str: + if idx == total - 1: + inner = body + fn = "array_check" + else: + child_var = _iter_var_name(idx + 1, total) + child_accessor = _element_accessor(var, inner_struct_paths[idx]) + inner = frame(idx + 1, child_accessor, child_var) + fn = "nested_array_check" + if idx == 0 and gate_parts: + inner = _wrap_element_gate(inner, var, gate_parts) + return f"{fn}({accessor}, lambda {var}: {inner})" + + return frame(0, f'"{column_path}"', "el") + + +def _render_array_check_expr( + target: ArrayPath, + desc: ExpressionDescriptor, + *, + element_guards: tuple[ElementGuard, ...] = (), + gate_parts: tuple[str, ...] = (), +) -> str: + """Render an ArrayPath target to an array_check / nested_array_check expression. + + Element guards are applied at the innermost iteration variable. This + assumes each guard's discriminator lives on the same struct level as + the leaf accessor -- which is true today because `ElementGuard`s only + arise from a union variant whose discriminator field is the + immediately enclosing array element. A future case where a check is + reached through further iteration *inside* a discriminated union + element would need per-guard depth info to apply the guard at the + correct frame. + """ + inner_struct_paths = target.iter_struct_paths + iteration_count = 1 + len(inner_struct_paths) + + innermost_var = _iter_var_name(iteration_count - 1, iteration_count) + leaf_accessor = _element_accessor(innermost_var, target.leaf) + body = _render_expr_call(desc, leaf_accessor) + + for guard in reversed(element_guards): + body = _render_variant_expr( + body, guard.values, guard.discriminator, in_array=True, var=innermost_var + ) + + return _wrap_in_array_iteration( + target.column_path, inner_struct_paths, body, gate_parts=gate_parts + ) + + +def _map_iter_var(projection: MapProjection) -> str: + """Lambda variable name for a map projection: `k` for keys, `v` for values.""" + return "k" if projection is MapProjection.KEY else "v" + + +def _wrap_in_map_iteration(target: MapPath, body: str) -> str: + """Wrap `body` in a map_keys_check / map_values_check projection lambda. + + The map helper projects the map (`F.map_keys` / `F.map_values`) and + applies the lambda to each projected key or value, the map analogue of + `_wrap_in_array_iteration`. `body` references the projected element via + the same `_map_iter_var(target.projection)` name this builds the lambda + parameter from. + """ + helper = map_runtime_helper(target.projection) + var = _map_iter_var(target.projection) + return f'{helper}("{target.map_column}", lambda {var}: {body})' + + +def _render_map_check_expr(target: MapPath, desc: ExpressionDescriptor) -> str: + """Render a MapPath target to a map_keys_check / map_values_check expression. + + A non-empty `target.leaf` navigates into a `dict[K, Model]` value struct + (`v["field"]`), mirroring an array element's leaf accessor; an empty leaf + applies the check to the projected scalar itself. + """ + var = _map_iter_var(target.projection) + body = _render_expr_call(desc, _element_accessor(var, target.leaf)) + return _wrap_in_map_iteration(target, body) + + +def _render_variant_expr( + inner_expr: str, + variant_values: tuple[str, ...], + discriminator_field: str, + *, + in_array: bool = False, + var: str = "el", +) -> str: + """Wrap an expression in F.when(...).isin() gating for union variant fields.""" + values_repr = py_literal(list(variant_values)) + disc_ref = ( + f'{var}["{discriminator_field}"]' + if in_array + else f'F.col("{discriminator_field}")' + ) + return f"F.when({disc_ref}.isin({values_repr}), {inner_expr})" + + +def _render_column_gate(expr: str, gate: FieldPath) -> str: + """Wrap an expression in F.when(gate.isNotNull(), ...) for nullable parent gating.""" + return f'F.when(F.col("{gate}").isNotNull(), {expr})' + + +def _model_check_func_name(check: ModelCheck, idx: int) -> str: + """Build the private function name for a model-constraint check. + + Array and map targets prefix the column path -- using the full encoded + `FieldPath` when the check is reached via inner iteration or leaf struct + navigation, otherwise the outer column name alone -- so collisions + across nested contexts get distinct identifiers. Row-root targets emit + `_{fn}_{idx}_check`. + """ + fn = model_constraint_function(check.descriptor) + match check.target: + case ArrayPath() as target: + has_nested_path = bool(target.iter_struct_paths) or bool(target.leaf) + prefix_source = str(target) if has_nested_path else target.column_path + prefix = sanitize_field_name(prefix_source) + return f"_{prefix}_{fn}_{idx}_check" + case MapPath() as target: + prefix_source = str(target) if target.leaf else target.map_column + prefix = sanitize_field_name(prefix_source) + return f"_{prefix}_{fn}_{idx}_check" + case _: + return f"_{fn}_{idx}_check" + + +def _check_shape_token(target: FieldPath) -> str: + """Token naming the runtime `CheckShape` member for a target path. + + Mirrors the member names of `overture.schema.pyspark.check.CheckShape`; + the check-function template prefixes `CheckShape.` to the result. An + `ArrayPath` or `MapPath` target renders to an `array` + expression (the map helper iterates the projected keys/values), every + other path to a nullable string. + """ + return "ARRAY" if isinstance(target, (ArrayPath, MapPath)) else "SCALAR" + + +def _render_check_expr(check: Check, descriptor_idx: int) -> str: + """Render the PySpark expression for one descriptor of `check`.""" + desc = check.descriptors[descriptor_idx] + column_guards = tuple(g for g in check.guards if isinstance(g, ColumnGuard)) + element_guards = tuple(g for g in check.guards if isinstance(g, ElementGuard)) + + match check.target: + case ScalarPath(): + expr = _render_expr_call(desc, f'F.col("{check.target}")') + if desc.gate: + expr = _render_column_gate(expr, desc.gate) + case ArrayPath(): + gate_parts: tuple[str, ...] = () + if desc.gate is not None: + # check_builder zeros the nullable gate when descending into + # a list (see `_recurse_into_model`), so a gate paired with + # an ArrayPath target should never occur today. If it does, + # the column-level fallback below would silently hide a + # codegen bug -- raise instead. + element_relative = check.target.element_relative_gate(desc.gate) + if element_relative is None: + raise AssertionError( + f"ArrayPath target with column-level gate is not " + f"produced by check_builder (gate={desc.gate!r}, " + f"target={check.target!r})" + ) + gate_parts = element_relative + expr = _render_array_check_expr( + check.target, + desc, + element_guards=element_guards, + gate_parts=gate_parts, + ) + case MapPath(): + expr = _render_map_check_expr(check.target, desc) + case _: + raise TypeError( + f"Unhandled FieldPath variant: {type(check.target).__name__}" + ) + + for guard in reversed(column_guards): + expr = _render_variant_expr(expr, guard.values, guard.discriminator) + return expr + + +def _check_function_context( + *, + target: FieldPath, + func_name: str, + field: str, + name: str, + expr: str, + read_columns: frozenset[str], +) -> dict[str, object]: + """Assemble the template context dict for one check function.""" + return { + "func_name": func_name, + "field": field, + "check_name": name, + "expr": expr, + "shape": _check_shape_token(target), + "read_columns": read_columns, + } + + +def _render_check_function_context(row: FieldCheckRow) -> dict[str, object]: + """Build the template context for a per-field check function from a row. + + The row carries the final `func_name`, `label`, and `name`; the + collisions that produce them are resolved once in `field_check_rows`. + """ + return _check_function_context( + target=row.check.target, + func_name=row.func_name, + field=row.label, + name=row.name, + expr=_render_check_expr(row.check, row.descriptor_idx), + read_columns=row.check.read_columns, + ) + + +def _render_model_constraint_function_context(row: ModelCheckRow) -> dict[str, object]: + """Build the template context for a model-constraint check function.""" + check = row.check + desc = check.descriptor + target = check.target + match target: + case ArrayPath(): + in_array = True + var = "inner" if target.iter_struct_paths else "el" + struct_path: tuple[str, ...] = target.leaf + case MapPath(): + # The map's values are iterated like an array element, so field + # references use the element accessor (`v["foo"]`) under the + # projected variable. + in_array = True + var = _map_iter_var(target.projection) + struct_path = target.leaf + case _: + in_array = False + var, struct_path = "el", () + + def _field_ref(field_name: str) -> str: + return _render_field_ref( + field_name, in_array=in_array, struct_path=struct_path, var=var + ) + + fn = model_constraint_function(desc) + + def _cols_and_names(field_names: tuple[str, ...]) -> tuple[str, str]: + cols_list = "[" + ", ".join(_field_ref(f) for f in field_names) + "]" + names_list = py_literal(list(field_names)) + return cols_list, names_list + + match desc: + case RequireAnyOf() | RadioGroup(): + cols_list, names_list = _cols_and_names(desc.field_names) + inner_expr = f"{fn}({cols_list}, {names_list})" + case RequireAnyTrue(): + parsed_conditions = [require_field_eq(c) for c in desc.conditions] + conds_list = ( + "[" + + ", ".join( + _render_condition( + p, in_array=in_array, struct_path=struct_path, var=var + ) + for p in parsed_conditions + ) + + "]" + ) + names_list = py_literal([p.field_name for p in parsed_conditions]) + inner_expr = f"{fn}({conds_list}, {names_list})" + case RequireIf() | ForbidIf(): + target_name = desc.field_names[0] + parsed = require_field_eq(desc.condition) + condition_expr = _render_condition( + parsed, in_array=in_array, struct_path=struct_path, var=var + ) + condition_desc = _render_condition_desc(parsed) + target_ref = _field_ref(target_name) + inner_expr = ( + f"{fn}({target_ref}, {condition_expr}, {py_literal(condition_desc)})" + ) + case MinFieldsSet(): + cols_list, names_list = _cols_and_names(desc.field_names) + inner_expr = f"{fn}({cols_list}, {names_list}, {desc.count})" + case _: + raise TypeError(f"Unhandled model constraint descriptor: {desc!r}") + + if isinstance(target, ArrayPath): + if check.gate is not None: + assert not target.iter_struct_paths, ( + f"gated ModelCheck with a nested-array target ({target!r}) is unsupported; " + f"the element-gate wrap assumes a single array level" + ) + element_relative = target.element_relative_gate(check.gate) + assert element_relative is not None, ( + f"ModelCheck gate={check.gate!r} is not reachable as an element-level " + f"accessor on target={target!r}; gates on ModelChecks must be ArrayPaths " + f"entering the same outer array as the target" + ) + inner_expr = _wrap_element_gate(inner_expr, var, element_relative) + expr = _wrap_in_array_iteration( + target.column_path, target.iter_struct_paths, inner_expr + ) + elif isinstance(target, MapPath): + # A `dict[K, Model]` value-model constraint wraps in map_values_check + # (or map_keys_check), iterating the projected values like an array. + # check_builder zeros the gate for iterated containers, so no gate + # reaches here. + assert check.gate is None, ( + f"ModelCheck gate={check.gate!r} paired with MapPath target={target!r}; " + f"map iteration handles value nullability, so a gate is unexpected" + ) + expr = _wrap_in_map_iteration(target, inner_expr) + else: + assert check.gate is None, ( + f"ModelCheck gate={check.gate!r} paired with non-ArrayPath target={target!r}; " + f"a gate only makes sense when the constrained model is inside an array" + ) + expr = inner_expr + + return _check_function_context( + target=target, + func_name=_model_check_func_name(check, row.idx), + field=row.label, + name=row.name, + expr=expr, + read_columns=check.read_columns, + ) + + +def _collect_constraint_expr_imports( + field_checks: list[Check], + model_checks: list[ModelCheck], +) -> set[str]: + """Collect all constraint_expressions function names needed. + + Field-descriptor names go through a `_COLUMN_PATTERN_HELPERS` + filter so column-pattern helpers route to their own import bucket. + Model-constraint function names (`check_require_any_of`, + `check_radio_group`, ...) are disjoint from that set, so they pass + through unfiltered. + """ + names: set[str] = set() + for check in field_checks: + for desc in check.descriptors: + if desc.function not in _COLUMN_PATTERN_HELPERS: + names.add(desc.function) + if desc.allow_literals: + names.add("except_literals") + for mc in model_checks: + names.add(model_constraint_function(mc.descriptor)) + return names + + +def _needs_geometry_type_import(field_checks: list[Check]) -> bool: + """Return True when any descriptor arg is a GeometryType instance.""" + for check in field_checks: + for desc in check.descriptors: + if any(isinstance(a, GeometryType) for a in desc.args): + return True + return False + + +def _pattern_imports_for(target: FieldPath) -> set[str]: + """Column-pattern helpers needed to iterate `target`.""" + match target: + case ArrayPath(): + names = {"array_check"} + if target.iter_struct_paths: + names.add("nested_array_check") + return names + case MapPath(): + return {map_runtime_helper(target.projection)} + case _: + return set() + + +def _collect_column_pattern_imports( + field_checks: list[Check], + model_checks: list[ModelCheck], +) -> set[str]: + """Collect column_patterns function names needed.""" + names: set[str] = set() + for check in field_checks: + names |= _pattern_imports_for(check.target) + for desc in check.descriptors: + if desc.function in _COLUMN_PATTERN_HELPERS: + names.add(desc.function) + for mc in model_checks: + names |= _pattern_imports_for(mc.target) + return names + + +_IDENTIFIER_TOKEN = re.compile(r"[A-Z][A-Za-z0-9_]*") + + +def _identifier_tokens(expr: str) -> set[str]: + """Tokenize a Spark type expression into capitalized identifiers.""" + return set(_IDENTIFIER_TOKEN.findall(expr)) + + +def _collect_spark_type_imports(schema_fields: list[SchemaField]) -> set[str]: + """Collect Spark type class names from schema field type expressions. + + `StructType` and `StructField` are always included: the model module + template emits the schema constant as `StructType([...])` unconditionally, + so the import must be present even when there are no fields. + """ + used: set[str] = {"StructType", "StructField"} + for sf in schema_fields: + used |= _identifier_tokens(sf.type_expr) & _SPARK_TYPES + return used + + +def _collect_schema_struct_imports(schema_fields: list[SchemaField]) -> set[str]: + """Collect _schema_structs constant names referenced in field type expressions.""" + refs: set[str] = set() + for sf in schema_fields: + refs |= _identifier_tokens(sf.type_expr) & _SHARED_STRUCT_REFS + return refs + + +def _field_check_function_entries( + field_checks: list[Check], +) -> list[dict[str, object]]: + """Build template contexts for field-level checks.""" + return [ + _render_check_function_context(row) for row in field_check_rows(field_checks) + ] + + +def _model_check_function_entries( + model_checks: list[ModelCheck], +) -> list[dict[str, object]]: + """Build template contexts for model-level checks.""" + return [ + _render_model_constraint_function_context(row) + for row in model_check_rows(model_checks) + ] + + +def render_model_module( + model_name: str, + field_checks: list[Check], + model_checks: list[ModelCheck], + schema_fields: list[SchemaField], + geometry_types: tuple[GeometryType, ...] = (), + *, + entry_point: str = "tests.placeholder:Placeholder", + partitions: Mapping[str, str] | None = None, +) -> str: + """Render a complete Python module for a model's checks and schema.""" + constraint_expr_fns = sorted( + _collect_constraint_expr_imports(field_checks, model_checks) + ) + column_pattern_fns = sorted( + _collect_column_pattern_imports(field_checks, model_checks) + ) + spark_types = sorted(_collect_spark_type_imports(schema_fields)) + schema_struct_refs = sorted(_collect_schema_struct_imports(schema_fields)) + geometry_type = _needs_geometry_type_import(field_checks) or bool(geometry_types) + geometry_types_literal = ( + _render_geometry_types(geometry_types) if geometry_types else None + ) + + check_functions = _field_check_function_entries( + field_checks + ) + _model_check_function_entries(model_checks) + + model_title = model_name.replace("_", " ").title() + + template = jinja_env().get_template("model_module.py.jinja2") + return template.render( + model_name=model_name, + model_title=model_title, + constraint_expr_fns=constraint_expr_fns, + column_pattern_fns=column_pattern_fns, + spark_types=spark_types, + schema_struct_refs=schema_struct_refs, + geometry_type=geometry_type, + check_functions=check_functions, + schema_const_name=schema_const_name(model_name), + schema_fields=schema_fields, + geometry_types_literal=geometry_types_literal, + entry_point=entry_point, + partitions=dict(partitions) if partitions else {}, + ) + + +def _render_geometry_types(geo: tuple[GeometryType, ...]) -> str: + """Render a `geometry_types` tuple literal. + + `GeometryType` is an Enum, so `repr()` does not produce a valid + expression -- members need explicit `GeometryType.NAME` source. + """ + return tuple_literal(_geometry_type_literal(g) for g in geo) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/schema_builder.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/schema_builder.py new file mode 100644 index 000000000..08469d67c --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/schema_builder.py @@ -0,0 +1,201 @@ +"""Build StructType schema source from ModelSpec field trees.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from ..extraction.field import ( + AnyScalar, + ArrayOf, + FieldShape, + LiteralScalar, + MapOf, + ModelRef, + NewTypeShape, + Primitive, + Scalar, + UnionRef, +) +from ..extraction.field_walk import enum_source, terminal_scalar +from ..extraction.specs import FieldSpec, ModelSpec, UnionSpec +from ..extraction.type_registry import get_type_mapping + +__all__ = [ + "SHARED_TYPE_REFS", + "SchemaField", + "build_schema", + "spark_type_rank", +] + +# Types whose base_type name maps to a _schema_structs.py StructType constant. +# Reserved for types the codegen cannot walk (BBox is a plain class, not a +# Pydantic BaseModel). Pydantic BaseModels are inlined. +SHARED_TYPE_REFS: dict[str, str] = { + "BBox": "BBOX_STRUCT", +} + +# Literal and Enum fields both serialize as strings in Parquet. +_STRING_FALLBACK = "StringType()" + + +@dataclass(frozen=True, slots=True) +class SchemaField: + """One field in the generated StructType. + + Parameters + ---------- + name + Column name. + type_expr + Spark type expression string (e.g. `"StringType()"`) or + a `_schema_structs.py` constant name. + """ + + name: str + type_expr: str + + +def _spark_for_base(base_type: str, source_type: type | None) -> str: + """Return a Spark type expression for a primitive base type. + + Tries `base_type` first, then falls back to `source_type.__name__`. + Returns `StringType()` when neither maps to a known Spark type. + """ + mapping = get_type_mapping(base_type) + if mapping is not None and mapping.spark is not None: + return mapping.spark + if source_type is not None: + fallback = get_type_mapping(source_type.__name__) + if fallback is not None and fallback.spark is not None: + return fallback.spark + return _STRING_FALLBACK + + +def _spark_for_scalar(scalar: Scalar) -> str: + """Map a `Scalar` variant to a Spark type expression. + + `LiteralScalar` and `AnyScalar` serialize as strings. `Primitive` + scalars look up the type registry; enum primitives and BBox short- + circuit to strings / shared constants before the registry. + """ + if isinstance(scalar, (LiteralScalar, AnyScalar)): + return _STRING_FALLBACK + if scalar.base_type in SHARED_TYPE_REFS: + return SHARED_TYPE_REFS[scalar.base_type] + if enum_source(scalar) is not None: + return _STRING_FALLBACK + return _spark_for_base(scalar.base_type, scalar.source_type) + + +# Spark numeric type widening precedence (higher rank = wider type). +_SPARK_TYPE_WIDENING: dict[str, int] = { + "IntegerType()": 0, + "LongType()": 1, + "DoubleType()": 2, +} + + +def spark_type_rank(field_spec: FieldSpec) -> int: + """Return a widening rank for the field's resolved Spark type. + + Fields with a higher rank are preferred when deduplicating union + members by name. Non-numeric types return -1 (no widening). + """ + scalar = terminal_scalar(field_spec.shape) + if not isinstance(scalar, Primitive): + return -1 + expr = _spark_for_base(scalar.base_type, scalar.source_type) + return _SPARK_TYPE_WIDENING.get(expr, -1) + + +def _deduplicate_by_name(fields: list[FieldSpec]) -> list[FieldSpec]: + """Keep one FieldSpec per name, widening the Spark type on conflict. + + Union annotated_fields may contain the same field name with different + type shapes (e.g. `value` as uint8 in one variant and float64 in + another). Parquet stores one column per name, so the schema needs + exactly one entry. When two fields share a name, the one with the + wider numeric Spark type wins (matching Parquet's type-widening + behavior). Two same-named fields whose Spark types are non-numeric and + not identical cannot share a column, so the collision fails loudly + rather than silently keeping whichever arm came first. + """ + seen: dict[str, FieldSpec] = {} + for f in fields: + existing = seen.get(f.name) + if existing is None: + seen[f.name] = f + continue + rank_f, rank_existing = spark_type_rank(f), spark_type_rank(existing) + if rank_f < 0 and rank_existing < 0: + spark_f = _shape_to_spark(f.shape) + spark_existing = _shape_to_spark(existing.shape) + if spark_f != spark_existing: + raise ValueError( + f"Union field {f.name!r} resolves to incompatible " + f"non-widening Spark types across arms " + f"({spark_existing} vs {spark_f}); a single Parquet " + "column cannot represent both." + ) + if rank_f > rank_existing: + seen[f.name] = f + return list(seen.values()) + + +def _struct_type_expr(fields: list[FieldSpec]) -> str: + """Build an inline `StructType([...])` expression from a list of fields.""" + if not fields: + raise ValueError( + "Cannot build a StructType for a model with no fields; an empty " + "struct column cannot carry data and signals an upstream " + "extraction problem." + ) + parts = [ + f'StructField("{f.name}", {_shape_to_spark(f.shape)}, True)' for f in fields + ] + return f"StructType([{', '.join(parts)}])" + + +def _shape_to_spark(shape: FieldShape) -> str: + """Convert a FieldShape to a Spark type expression string.""" + match shape: + case ArrayOf(element=element): + return f"ArrayType({_shape_to_spark(element)}, True)" + case NewTypeShape(inner=inner): + return _shape_to_spark(inner) + case ModelRef(model=m): + return _struct_type_expr(m.fields) + case UnionRef(union=u): + return _struct_type_expr(_deduplicate_by_name(u.fields)) + case MapOf(key=k, value=v): + return f"MapType({_shape_to_spark(k)}, {_shape_to_spark(v)}, True)" + case Primitive() | LiteralScalar() | AnyScalar() as s: + return _spark_for_scalar(s) + raise TypeError(f"Unhandled FieldShape: {shape!r}") + + +def build_schema(spec: ModelSpec) -> list[SchemaField]: + """Build schema fields for a feature spec. + + Walks the field tree and maps types to Spark type expressions. + Recognizes shared types and emits fields in model order. + + Parameters + ---------- + spec + The feature spec to build schema fields for. + + Returns + ------- + list[SchemaField] + One entry per schema column in model order. + """ + source_fields = ( + _deduplicate_by_name(spec.fields) + if isinstance(spec, UnionSpec) + else spec.fields + ) + return [ + SchemaField(name=f.name, type_expr=_shape_to_spark(f.shape)) + for f in source_fields + ] diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/templates/_check_function.py.jinja2 b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/templates/_check_function.py.jinja2 new file mode 100644 index 000000000..beef4fdab --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/templates/_check_function.py.jinja2 @@ -0,0 +1,10 @@ +{%- macro check_function(c) -%} +def {{ c.func_name }}() -> Check: + return Check( + field={{ c.field | py_literal }}, + name={{ c.check_name | py_literal }}, + expr={{ c.expr }}, + shape=CheckShape.{{ c.shape }}, + read_columns={{ c.read_columns | py_literal }}, + ) +{% endmacro %} diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/templates/model_module.py.jinja2 b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/templates/model_module.py.jinja2 new file mode 100644 index 000000000..8c91646c4 --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/templates/model_module.py.jinja2 @@ -0,0 +1,83 @@ +{% from '_check_function.py.jinja2' import check_function -%} +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""{{ model_title }} validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +{% if spark_types %} +from pyspark.sql.types import ( +{% for t in spark_types %} + {{ t }}, +{% endfor %} +) +{% endif %} +{% if geometry_type %} +from overture.schema.system.primitive import GeometryType + +{% endif %} +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +{% if schema_struct_refs %} +from overture.schema.pyspark.expressions._schema_structs import ( +{% for r in schema_struct_refs %} + {{ r }}, +{% endfor %} +) +{% endif %} +{% if column_pattern_fns %} +from overture.schema.pyspark.expressions.column_patterns import ( +{% for f in column_pattern_fns %} + {{ f }}, +{% endfor %} +) +{% endif %} +{% if constraint_expr_fns %} +from overture.schema.pyspark.expressions.constraint_expressions import ( +{% for f in constraint_expr_fns %} + {{ f }}, +{% endfor %} +) +{% endif %} + + +{% for c in check_functions %} +{{ check_function(c) }} +{% endfor %} + +def {{ model_name }}_checks() -> list[Check]: + """All validation checks for {{ model_name }}.""" +{% if check_functions %} + return [ +{% for c in check_functions %} + {{ c.func_name }}(), +{% endfor %} + ] +{% else %} + return [] +{% endif %} + + +{{ schema_const_name }} = StructType( + [ +{%- for sf in schema_fields %} + StructField("{{ sf.name }}", {{ sf.type_expr }}, True), +{%- endfor %} + ] +) +{% if geometry_types_literal %} + +GEOMETRY_TYPES: tuple[GeometryType, ...] = {{ geometry_types_literal }} +{% endif %} + +ENTRY_POINT = "{{ entry_point }}" + +PARTITIONS: dict[str, str] = {{ partitions | py_literal }} + +MODEL_VALIDATION = ModelValidation( + schema={{ schema_const_name }}, + checks={{ model_name }}_checks, +{%- if geometry_types_literal %} + geometry_types=GEOMETRY_TYPES, +{%- endif %} +) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/templates/test_module.py.jinja2 b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/templates/test_module.py.jinja2 new file mode 100644 index 000000000..466103cf2 --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/templates/test_module.py.jinja2 @@ -0,0 +1,129 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for {{ model_name }}.""" + +from __future__ import annotations + +import pytest +from {{ expression_import }} import ( + {{ model_name | upper }}_SCHEMA, + {{ model_name }}_checks, +) +from pyspark.sql import SparkSession + +from {{ support_prefix }}_support.harness import ( + ValidationResults, + run_validation_pipeline, +) +{% if mutation_imports %} +from {{ support_prefix }}_support.mutations import {{ mutation_imports | join(", ") }} +{% endif %} +{% if needs_set_at_path %} +from {{ support_prefix }}_support.helpers import set_at_path +{% endif %} +from {{ support_prefix }}_support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = {{ base_row_sparse }} + + +BASE_ROW_POPULATED: dict = {{ base_row_populated }} + + +SCENARIOS: list[Scenario] = [ +{% for entry in scenarios %} + Scenario( + {% for k, v in entry %} + {{ k }}={{ v }}, + {% endfor %} + ), +{% endfor %} +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return {{ model_name }}_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + {{ schema_name }}, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="{{ model_name }}", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + {{ schema_name }}, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="{{ model_name }}", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("{{ model_name }}::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("{{ model_name }}::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/__init__.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/__init__.py new file mode 100644 index 000000000..fa271d7b8 --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/__init__.py @@ -0,0 +1,9 @@ +"""Test-data generation for the rendered PySpark conformance tests. + +Three modules cover three flavors of data: + +- `invalid_value`: constraint-violating values for triggering each check. +- `base_row`: minimal and fully populated valid rows. +- `scaffold`: sparse path scaffolds that supply the nested intermediates + (optional structs, arrays) a check's field path requires. +""" diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/base_row.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/base_row.py new file mode 100644 index 000000000..70a20e0ee --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/base_row.py @@ -0,0 +1,822 @@ +"""Generate valid base rows for the rendered conformance tests. + +`generate_base_row` produces a minimal valid row (required fields only) +from a `ModelSpec`. `generate_populated_row` produces a fully +populated row including optional fields. `generate_arm_rows` and +`generate_populated_arm_rows` do the same for each arm of a discriminated +union. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Callable +from enum import Enum +from typing import Any + +from overture.schema.common.scoping.lr import LinearReferenceRangeConstraint +from overture.schema.system.model_constraint import ( + ForbidIfConstraint, + MinFieldsSetConstraint, + RadioGroupConstraint, + RequireAnyOfConstraint, + RequireAnyTrueConstraint, + RequireIfConstraint, +) +from overture.schema.system.primitive.geom import ( + Geometry, + GeometryType, + GeometryTypeConstraint, +) + +from ...extraction.field import ( + AnyScalar, + ArrayOf, + ConstraintSource, + FieldShape, + LiteralScalar, + MapOf, + ModelRef, + NewTypeShape, + Primitive, + UnionRef, +) +from ...extraction.field_walk import ( + enum_source, + has_array_layer, + terminal_primitive, + terminal_scalar, +) +from ...extraction.length_constraints import ArrayMinLen +from ...extraction.specs import FieldSpec, ModelSpec, RecordSpec, UnionSpec +from ...extraction.type_registry import primitive_spark_category +from .._primitive_fill import PRIMITIVE_FILL_TABLE +from ..constraint_dispatch import ( + ExpressionDescriptor, + FieldEq, + dispatch_constraint, + require_bool_field_eq, + require_field_eq, +) +from ..schema_builder import spark_type_rank +from .constraint_values import ( + CONSTRAINT_VALUES, + curated_pattern_values, + uncurated_pattern_error, + valid_bound, +) + +__all__ = [ + "condition_overrides_for_present_field", + "generate_arm_rows", + "generate_base_row", + "generate_populated_arm_rows", + "generate_populated_row", + "resolve_arm_spec", + "value_for_field", +] + +_BASE_ROW_NAMESPACE = uuid.uuid5( + uuid.NAMESPACE_DNS, "overturemaps.org/codegen/base_row" +) + + +# WKT strings for each allowed geometry type (valid side) +_VALID_GEOMETRY_WKT: dict[GeometryType, str] = { + GeometryType.POINT: "POINT (0 0)", + GeometryType.LINE_STRING: "LINESTRING (0 0, 1 1)", + GeometryType.POLYGON: "POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))", + GeometryType.MULTI_POLYGON: "MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", + GeometryType.MULTI_LINE_STRING: "MULTILINESTRING ((0 0, 1 1))", +} + + +_PRIMITIVE_DEFAULTS: dict[str, object] = { + "str": "", + "NoWhitespaceString": "", + "HttpUrl": "https://example.com/", + "EmailStr": "user@example.com", + "bool": False, + "bytes": b"", + "datetime": "2024-01-01T00:00:00Z", + "date": "2024-01-01", +} + + +def _bbox_value() -> dict[str, float]: + return {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0} + + +# Field-name overrides applied before any shape-based value generation in +# `value_for_field`. Each builder receives `(field_spec, spec_name)`. +_SPECIAL_FIELD_VALUES: dict[str, Callable[[FieldSpec, str], object]] = { + "id": lambda _f, spec_name: str(uuid.uuid5(_BASE_ROW_NAMESPACE, spec_name)), + "bbox": lambda _f, _spec_name: _bbox_value(), +} + + +def _is_geometry_terminal(terminal: Primitive) -> bool: + """Whether this terminal represents a geometry value. + + Only fires for the `Geometry` source class. Fields wanting a + geometry value must declare `Geometry`; ad-hoc forms like + `Annotated[bytes, GeometryTypeConstraint(...)]` aren't recognized. + """ + return terminal.source_type is Geometry + + +def generate_base_row(spec: ModelSpec, *, index: int = 0) -> dict[str, Any]: + """Produce a minimal valid row from a feature spec (required fields only). + + The row passes `TypeAdapter(validation_type).validate_python()`. + + Parameters + ---------- + spec + An expanded feature spec. + index + Position within a parent list. Non-zero values suffix string fields + to ensure uniqueness across list items. + """ + return _build_row(spec, index=index, populate_optional=False) + + +def generate_populated_row(spec: ModelSpec, *, index: int = 0) -> dict[str, Any]: + """Produce a fully populated valid row (all fields, including optional). + + Sub-models are recursively populated. + + Parameters + ---------- + spec + An expanded feature spec. + index + Position within a parent list. Non-zero values suffix string fields + to ensure uniqueness across list items. + """ + return _build_row(spec, index=index, populate_optional=True) + + +def generate_arm_rows(spec: ModelSpec) -> dict[str, dict[str, Any]]: + """Produce one minimal valid row per discriminator arm of a union. + + Returns `{arm_value: row}` where each row passes TypeAdapter + validation against the union's source annotation. + + Parameters + ---------- + spec + An expanded union spec. + """ + return _build_arm_rows(_require_union(spec), populate_optional=False) + + +def generate_populated_arm_rows( + spec: ModelSpec, +) -> dict[str, dict[str, Any]]: + """Produce one fully populated valid row per discriminator arm. + + Returns `{arm_value: row}` where each row passes TypeAdapter + validation and includes all optional fields with valid values. + + Parameters + ---------- + spec + An expanded union spec. + """ + return _build_arm_rows(_require_union(spec), populate_optional=True) + + +def _require_union(spec: ModelSpec) -> UnionSpec: + if not isinstance(spec, UnionSpec): + raise TypeError( + f"Expected a UnionSpec, got {type(spec).__name__}: {spec.name!r}" + ) + return spec + + +def _build_row( + spec: ModelSpec, + *, + index: int = 0, + populate_optional: bool, + name_override: str | None = None, +) -> dict[str, Any]: + row: dict[str, Any] = {} + name = name_override or spec.name + for field in spec.fields: + if not populate_optional and not field.is_required: + continue + row[field.name] = value_for_field( + field, name, index=index, populate_optional=populate_optional + ) + _satisfy_model_constraints(row, spec) + return row + + +def _build_arm_rows( + spec: UnionSpec, + *, + populate_optional: bool, +) -> dict[str, dict[str, Any]]: + if spec.discriminator_field is None or spec.discriminator_mapping is None: + raise ValueError(f"UnionSpec {spec.name!r} has no discriminator") + if spec.constraints: + # Per-arm rows are built from member specs only; union-level + # constraints (e.g. radio_group on the union itself) would need + # `_satisfy_model_constraints` applied with the union's field + # list. No schema exercises this today; raise so a future union + # that adds one fails loudly rather than producing invalid rows. + raise NotImplementedError( + f"UnionSpec {spec.name!r} has {len(spec.constraints)} model " + "constraint(s); per-arm row generation does not enforce them" + ) + spec_by_class = {ms.member_cls: ms.spec for ms in spec.member_specs} + result: dict[str, dict[str, Any]] = {} + for arm_val, member_cls in spec.discriminator_mapping.items(): + row = _build_row( + spec_by_class[member_cls], + populate_optional=populate_optional, + name_override=spec.name, + ) + row[spec.discriminator_field] = arm_val + result[arm_val] = row + return result + + +def _condition_value(field_eq: FieldEq) -> object: + """The condition's comparison value, with an `Enum` unwrapped to its value. + + Row dicts store raw scalar values, so an `Enum`-typed condition value is + compared and written as its underlying `.value`. + """ + value = field_eq.value + return value.value if isinstance(value, Enum) else value + + +def _row_satisfies_condition(row: dict[str, Any], condition: object) -> bool: + """Check whether the condition is satisfied by the row's current values. + + Handles `FieldEqCondition` and `Not(FieldEqCondition)`. Raises + `TypeError` for any other condition kind so new condition types fail + loudly rather than silently returning an incorrect result. + + Parameters + ---------- + row + Current row dict being built. + condition + A `Condition` from a `RequireIfConstraint` or `ForbidIfConstraint`. + """ + field_eq = require_field_eq(condition) # type: ignore[arg-type] + matches = row.get(field_eq.field_name) == _condition_value(field_eq) + return matches != field_eq.negated + + +def _satisfy_model_constraints(row: dict[str, Any], spec: ModelSpec) -> None: + """Adjust *row* so each model constraint is satisfied. + + `require_if`/`radio_group`/`require_any_of`/`min_fields_set` fill in + optional fields the constraint makes mandatory. `forbid_if` removes + fields the constraint excludes. Constraints whose guard predicate is + false (e.g. a `RequireIf` whose condition does not hold for the + current row) need no adjustment and pass through; any constraint type + not matched by an arm here is silently skipped, intentionally -- new + constraint kinds surface via `dispatch_model_constraint` (which + raises) rather than here. + """ + fields_by_name = {f.name: f for f in spec.fields} + for constraint in spec.constraints: + match constraint: + case RequireIfConstraint() if _row_satisfies_condition( + row, constraint.condition + ): + for field_name in constraint.field_names: + if field_name in row: + continue + field_spec = fields_by_name.get(field_name) + if field_spec is not None: + row[field_name] = value_for_field(field_spec, spec.name) + case RadioGroupConstraint() if not any( + row.get(fn) is True for fn in constraint.field_names + ): + for field_name in constraint.field_names: + if field_name in fields_by_name: + row[field_name] = True + break + case RequireAnyTrueConstraint() if not any( + _row_satisfies_condition(row, c) for c in constraint.conditions + ): + # Make the first condition hold by writing the boolean it tests + # for. `require_bool_field_eq` validates the positive-boolean + # invariant here too -- this path runs on the raw constraint and + # does not pass through `dispatch_model_constraint`. + field_eq = require_bool_field_eq(constraint.conditions[0]) # type: ignore[arg-type] + row[field_eq.field_name] = field_eq.value + case RequireAnyOfConstraint() if not any( + fn in row for fn in constraint.field_names + ): + for field_name in constraint.field_names: + field_spec = fields_by_name.get(field_name) + if field_spec is not None: + row[field_name] = value_for_field(field_spec, spec.name) + break + case ForbidIfConstraint() if _row_satisfies_condition( + row, constraint.condition + ): + for field_name in constraint.field_names: + row.pop(field_name, None) + case MinFieldsSetConstraint(count=count): + # Mirror Pydantic's `model_fields_set` semantics: every + # required field is "set" by the constructor, and counts + # alongside any non-null optional field. Required fields + # are always populated by the time we reach this branch, + # so satisfying `count` may need extra optional fills. + missing = count - sum(1 for f in spec.fields if f.name in row) + for opt_field in (f for f in spec.fields if not f.is_required): + if missing <= 0: + break + if opt_field.name in row: + continue + row[opt_field.name] = value_for_field(opt_field, spec.name) + missing -= 1 + + +def _condition_disabling_value(field_eq: FieldEq, field_spec: FieldSpec) -> object: + """Return a value for a condition field that makes the condition false. + + `FieldEqCondition(f, X)` holds when `f == X`, so a different enum member + disables it; a negated condition (`Not(...)`, true when `f != X`) is + disabled by `X` itself. Every condition in the schema gates on an enum + field, so a non-enum condition field raises rather than guess a value. + + Parameters + ---------- + field_eq + The unwrapped field-equality condition. + field_spec + Spec of the condition field, used to enumerate alternative values. + """ + forbidden = _condition_value(field_eq) + if field_eq.negated: + return forbidden + terminal = terminal_primitive(field_spec.shape) + enum_cls = enum_source(terminal) if terminal is not None else None + if enum_cls is None: + raise TypeError( + f"condition field {field_eq.field_name!r} is not enum-backed; " + "cannot derive a value that disables its forbid_if condition" + ) + for member in enum_cls: + if member.value != forbidden: + return member.value + raise ValueError( + f"enum {enum_cls.__name__} has no member other than {forbidden!r}; " + "cannot disable its forbid_if condition" + ) + + +def condition_overrides_for_present_field( + spec: ModelSpec, field_name: str +) -> dict[str, Any]: + """Return overrides that let `field_name` be present on a valid base row. + + A `forbid_if` whose condition the base row satisfies forbids `field_name`, + so a scaffold that sets the field yields a row Pydantic rejects. Flip each + such condition field to a value the forbid rejects -- which also satisfies + the symmetric `require_if` that then mandates the field -- and re-satisfy + the model constraints, since a flipped condition can newly require other + fields. Returns only the fields whose value differs from the base row; + `field_name` itself is set by the scaffold and is excluded. + + Returns `{}` when no `forbid_if` gates `field_name`, the common case. + + Parameters + ---------- + spec + The model whose constraints govern `field_name`. + field_name + A direct field of `spec` the scaffold needs to set. + """ + forbidding = [ + c + for c in spec.constraints + if isinstance(c, ForbidIfConstraint) and field_name in c.field_names + ] + if not forbidding: + return {} + base = generate_base_row(spec) + fields_by_name = {f.name: f for f in spec.fields} + flips: dict[str, Any] = {} + for constraint in forbidding: + if not _row_satisfies_condition(base, constraint.condition): + continue + field_eq = require_field_eq(constraint.condition) + cond_field = fields_by_name.get(field_eq.field_name) + if cond_field is not None: + flips[field_eq.field_name] = _condition_disabling_value( + field_eq, cond_field + ) + if not flips: + return {} + merged = {**base, **flips} + _satisfy_model_constraints(merged, spec) + return { + name: value + for name, value in merged.items() + if name != field_name and base.get(name) != value + } + + +def value_for_field( + field: FieldSpec, + spec_name: str, + *, + index: int = 0, + populate_optional: bool = False, +) -> object: + """Produce a valid value for a single field. + + Consults field constraints via `dispatch_constraint` to produce + constraint-satisfying values (e.g., a valid country code instead of + an empty string). + + Parameters + ---------- + field + The field spec to produce a value for. + spec_name + The name of the containing spec, used for deterministic UUID generation. + index + Position within a parent list. Non-zero values suffix string fields + to ensure uniqueness across list items. + populate_optional + When True, MODEL and UNION sub-rows include optional fields via + `generate_populated_row`. When False (default), sub-rows are sparse + via `generate_base_row`. + """ + special = _SPECIAL_FIELD_VALUES.get(field.name) + if special is not None: + return special(field, spec_name) + + shape = field.shape + + # Geometry fields short-circuit to a WKT literal. PySpark's Geometry + # validator parses WKT via `from_wkt`; the field is stored as + # BinaryType (WKB) downstream. + terminal = terminal_primitive(shape) + if terminal is not None and _is_geometry_terminal(terminal): + return _geometry_wkt_from_shape_constraints(terminal.constraints) + + # Non-list fields: try a constraint-driven value (e.g. CountryCode -> "US") + # before falling back to type defaults. The terminal scalar carries the + # constraints directly in the no-list case. Lists go through the recursive + # shape walk so array-level constraints and per-element constraints both + # get a chance to drive value generation. + if not has_array_layer(shape) and terminal is not None: + constraint_val = _value_from_scalar_constraints(terminal) + if constraint_val is not None: + if index > 0 and isinstance(constraint_val, str): + return f"{constraint_val}{index}" + return constraint_val + + return _value_for_shape( + shape, + index=index, + check_constraints=False, + populate_optional=populate_optional, + ) + + +def _widest_union_member(union: UnionSpec) -> RecordSpec: + """Pick the union member whose fields have the highest cumulative Spark type rank. + + When multiple union members share a field name with different numeric + types (e.g. `value: uint8` in one variant and `value: float64` in + another), PySpark widens the column to the broadest type (DoubleType). + Generating a row from the narrower member produces Python `int` values + that PySpark silently converts to null in `DoubleType` columns. + + By selecting the member with the widest field types, the generated row + uses Python `float` values that PySpark accepts in `DoubleType` columns. + """ + best_spec = union.member_specs[0].spec + best_rank = -1 + for member in union.member_specs: + field_ranks = [spark_type_rank(f) for f in member.spec.fields] + rank = sum(r for r in field_ranks if r >= 0) + if rank > best_rank: + best_rank = rank + best_spec = member.spec + return best_spec + + +def resolve_arm_spec( + union: UnionSpec, discriminator_value: object | None = None +) -> RecordSpec: + """Return the member `RecordSpec` for one arm of a discriminated union. + + Without a discriminator value (a check not gated to a specific arm), + returns the widest member -- the one whose float types survive PySpark + column widening, per `_widest_union_member`. With a value, returns the + member that value selects, and raises when it selects none: a seeded + discriminator that matches no arm is a check_builder/scaffold inconsistency, + not a reason to fall back to an arm whose fields contradict the seed. + + Parameters + ---------- + union + The union to resolve an arm from. + discriminator_value + The discriminator value identifying the arm (e.g. a scaffold's seeded + `ElementGuard` value), matching a `discriminator_mapping` key. + + Raises + ------ + ValueError + When `discriminator_value` is given but selects no member arm. + """ + if discriminator_value is None: + return _widest_union_member(union) + mapping = union.discriminator_mapping or {} + member_cls = mapping.get(discriminator_value) # type: ignore[call-overload] + if member_cls is not None: + for member in union.member_specs: + if member.member_cls is member_cls: + return member.spec + raise ValueError( + f"discriminator {discriminator_value!r} selects no arm of union " + f"{union.name!r} (arms: {sorted(mapping)})" + ) + + +def _row_from_model_spec( + spec: RecordSpec, + *, + index: int = 0, + populate_optional: bool = False, +) -> dict[str, Any]: + """Generate a row dict from an already-extracted model spec.""" + if populate_optional: + return generate_populated_row(spec, index=index) + return generate_base_row(spec, index=index) + + +def _value_for_shape( + shape: FieldShape, + *, + index: int = 0, + check_constraints: bool = True, + populate_optional: bool = False, +) -> object: + """Produce a valid value from a `FieldShape`. + + Each shape layer carries its own constraints: `ArrayOf`'s + constraints drive list-length decisions; the element shape's + constraints (visible after descending into `element`) drive + per-item value generation. + + Parameters + ---------- + shape + The field shape to produce a value for. + index + Array element index, used to suffix strings for uniqueness. + check_constraints + When True, attempt constraint-driven value generation at the + terminal Scalar before falling back to a primitive default. + populate_optional + When True, MODEL and UNION sub-rows include optional fields via + `generate_populated_row`. When False (default), sub-rows are + sparse via `generate_base_row`. + """ + match shape: + case ArrayOf(element=element, constraints=array_constraints): + list_val = _list_value_from_shape_constraints(array_constraints) + if list_val is not None: + return list_val + count = _min_length_from_shape_constraints(array_constraints) + return [ + _value_for_shape(element, index=i, populate_optional=populate_optional) + for i in range(count) + ] + + case NewTypeShape(inner=inner): + return _value_for_shape( + inner, + index=index, + check_constraints=check_constraints, + populate_optional=populate_optional, + ) + + case MapOf(key=key_shape, value=value_shape): + # One constraint-valid entry: an empty map satisfies Pydantic + # but leaves nothing for a conformance scenario to corrupt, so + # the key/value checks would never fire. A `dict[K, Any]` value + # (e.g. Infrastructure.source_tags) carries no constraint -- and + # thus no check -- and `Any` has no value strategy, so the map + # stays empty: there is nothing to validate or corrupt. + if isinstance(terminal_scalar(value_shape), AnyScalar): + return {} + map_key = _value_for_shape( + key_shape, index=index, populate_optional=populate_optional + ) + map_value = _value_for_shape( + value_shape, index=index, populate_optional=populate_optional + ) + return {map_key: map_value} + + case LiteralScalar(values=values): + val = values[0] + return val.value if isinstance(val, Enum) else val + + case Primitive() as p if (enum_cls := enum_source(p)) is not None: + return list(enum_cls)[0].value + + case ModelRef(model=m): + return _row_from_model_spec( + m, index=index, populate_optional=populate_optional + ) + + case UnionRef(union=u): + # The selected member's discriminator field is a `Literal[X] = "x"` + # with a default, so it has `is_required=False`. In the populated + # case the LiteralScalar branch writes the literal explicitly; in + # the sparse case the field is omitted from the dict and Pydantic + # supplies the default during `TypeAdapter.validate_python()`. + return _row_from_model_spec( + _widest_union_member(u), + index=index, + populate_optional=populate_optional, + ) + + case AnyScalar(): + # No value strategy exists for `Any`. The map walk descends + # into key/value shapes, so a `dict[K, Any]` value would reach + # here -- no schema declares one today, and this raises loudly + # rather than guess a value if one ever appears. + raise TypeError( + "AnyScalar reached base-row generation; no value strategy exists" + ) + + case Primitive() as scalar: + constraint_val: object | None = None + if check_constraints: + constraint_val = _value_from_scalar_constraints(scalar) + val = ( + constraint_val + if constraint_val is not None + else _primitive_default(scalar.base_type) + ) + if index > 0 and isinstance(val, str): + val = f"{val}{index}" + return val + + raise TypeError(f"Unhandled FieldShape: {shape!r}") + + +def _value_from_check_enum( + desc: ExpressionDescriptor, _scalar: Primitive, _cs: ConstraintSource +) -> object: + """Return the first allowed value from a `check_enum` descriptor.""" + return desc.args[0][0] # type: ignore[index,no-any-return] + + +def _value_from_check_string_min_length( + _desc: ExpressionDescriptor, _scalar: Primitive, _cs: ConstraintSource +) -> str: + """Return any single character; satisfies `min_length>=1` for every schema today.""" + return "a" + + +def _value_from_check_pattern( + desc: ExpressionDescriptor, _scalar: Primitive, _cs: ConstraintSource +) -> object: + """Return a pattern-matching value for a curated raw pydantic pattern. + + Only raw `Field(pattern=)` constraints reach here -- named + `PatternConstraint` subclasses resolve earlier via `CONSTRAINT_VALUES`. + An uncurated pattern fails loud, symmetrically with `invalid_value`: + matching strings can't be generated generically, and silently falling + back to the primitive default would emit a row that fails the pattern, + surfacing later as a misleading "row should be valid" Pydantic error. + + Raises + ------ + ValueError + When the pattern has no curated entry in `PATTERN_VALUES`. + """ + curated = curated_pattern_values(desc) + if curated is None: + raise uncurated_pattern_error(desc, side="valid") + return curated.valid + + +# Builders for descriptor-driven values, keyed by `ExpressionDescriptor.function`. +# `check_bounds` is intentionally absent: it is routed through +# `_value_from_scalar_constraints` to merge multiple bound descriptors (e.g. +# separate Gt + Lt) before calling `valid_bound` once with the combined kwargs, +# so a single-bound path never silently produces a value that violates a second +# bound on the same field. +# `check_pattern` only yields a value for a curated raw pydantic pattern; +# named pattern constraints resolve earlier via `CONSTRAINT_VALUES`. +_DESCRIPTOR_VALUE_BUILDERS: dict[ + str, Callable[[ExpressionDescriptor, Primitive, ConstraintSource], object | None] +] = { + "check_enum": _value_from_check_enum, + "check_string_min_length": _value_from_check_string_min_length, + "check_pattern": _value_from_check_pattern, +} + + +_CONSTRAINT_VALID_LIST_VALUES: dict[type, list[object]] = { + LinearReferenceRangeConstraint: [0.0, 1.0], +} + + +def _value_from_scalar_constraints(scalar: Primitive) -> object | None: + """Return a value satisfying all dispatched constraints on a scalar. + + Maps known constraint types to valid values directly. For `check_bounds` + descriptors, merges all bound kwargs from every constraint on the field + into one dict and calls `valid_bound` once, so a field carrying separate + `Gt`/`Lt` constraints (two `check_bounds` descriptors) gets a value + satisfying both bounds. Non-bounds constraints use first-match behavior. + """ + merged_bounds: dict[str, object] = {} + for cs in scalar.constraints: + constraint_type = type(cs.constraint) + if constraint_type in CONSTRAINT_VALUES: + return CONSTRAINT_VALUES[constraint_type].valid + desc = dispatch_constraint(cs.constraint, base_type=scalar.base_type) + if desc is None: + continue + if desc.function == "check_bounds": + # Skip structural bounds from numeric NewType ranges — those are + # enforced by the Spark/Parquet type system, not by field constraints. + if cs.source_name != scalar.base_type: + merged_bounds.update(desc.kwargs) + continue + builder = _DESCRIPTOR_VALUE_BUILDERS.get(desc.function) + if builder is None: + continue + val = builder(desc, scalar, cs) + if val is not None: + return val + if merged_bounds: + merged_desc = ExpressionDescriptor( + function="check_bounds", kwargs=tuple(merged_bounds.items()) + ) + return valid_bound(merged_desc) + return None + + +def _list_value_from_shape_constraints( + constraints: tuple[ConstraintSource, ...], +) -> list[object] | None: + """Return a fixed valid list value if a list-level constraint requires it.""" + for cs in constraints: + val = _CONSTRAINT_VALID_LIST_VALUES.get(type(cs.constraint)) + if val is not None: + return val + return None + + +def _min_length_from_shape_constraints( + constraints: tuple[ConstraintSource, ...], +) -> int: + """Extract the array min_length from constraints anchored at this layer. + + Constraints sit on the `ArrayOf` whose iteration they govern, so any + `ArrayMinLen` we see here applies to this list level directly -- no + anchor arithmetic is required. + """ + for cs in constraints: + if isinstance(cs.constraint, ArrayMinLen): + return max(cs.constraint.min_length, 1) + return 1 + + +def _primitive_default(base_type: str) -> object: + """Return a type-appropriate default for a primitive base_type.""" + explicit = _PRIMITIVE_DEFAULTS.get(base_type) + if explicit is not None: + return explicit + category = primitive_spark_category(base_type) + entry = PRIMITIVE_FILL_TABLE.get(category) + return entry[1] if entry is not None else "" + + +def _geometry_wkt_from_shape_constraints( + constraints: tuple[ConstraintSource, ...], +) -> str: + """Extract the allowed geometry type from constraints and return valid WKT.""" + for cs in constraints: + if isinstance(cs.constraint, GeometryTypeConstraint): + geom_type = cs.constraint.allowed_types[0] + wkt = _VALID_GEOMETRY_WKT.get(geom_type) + if wkt is not None: + return wkt + raise ValueError(f"No WKT defined for geometry type: {geom_type!r}") + # No constraint — default to POINT + return _VALID_GEOMETRY_WKT[GeometryType.POINT] diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/constraint_values.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/constraint_values.py new file mode 100644 index 000000000..6a15a070e --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/constraint_values.py @@ -0,0 +1,203 @@ +"""Paired valid/invalid value generation for string-constraint and numeric-bound checks. + +Each entry in `CONSTRAINT_VALUES` carries both sides of the pair: +`valid` is accepted by the constraint; `invalid` violates it. +Both sides are mandatory — partial entries are not allowed. + +Consumed by `base_row` (uses the `valid` side to produce valid base rows) +and `invalid_value` (uses the `invalid` side to produce scenario mutations). + +`valid_bound` and `invalid_bound` are analogous functions for numeric +bound descriptors, placed here so both sides of every constraint kind +live in one module. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from overture.schema.system.field_constraint.string import ( + CountryCodeAlpha2Constraint, + HexColorConstraint, + JsonPointerConstraint, + LanguageTagConstraint, + NoWhitespaceConstraint, + PhoneNumberConstraint, + RegionCodeConstraint, + SnakeCaseConstraint, + StrippedConstraint, + WikidataIdConstraint, +) + +from ..constraint_dispatch import ExpressionDescriptor, normalize_anchor + +__all__ = [ + "CONSTRAINT_VALUES", + "PATTERN_VALUES", + "ConstraintValues", + "curated_pattern_values", + "invalid_bound", + "uncurated_pattern_error", + "valid_bound", +] + + +@dataclass(frozen=True, slots=True) +class ConstraintValues: + """A paired valid/invalid value for one constraint type.""" + + valid: object + invalid: object + + +CONSTRAINT_VALUES: dict[type, ConstraintValues] = { + CountryCodeAlpha2Constraint: ConstraintValues(valid="US", invalid="99"), + HexColorConstraint: ConstraintValues(valid="#aabbcc", invalid="not-hex"), + JsonPointerConstraint: ConstraintValues(valid="/valid/pointer", invalid="no-slash"), + LanguageTagConstraint: ConstraintValues(valid="en", invalid="123"), + NoWhitespaceConstraint: ConstraintValues( + valid="nowhitespace", invalid="has whitespace" + ), + PhoneNumberConstraint: ConstraintValues( + valid="+1 555-555-5555", invalid="1234567890" + ), + RegionCodeConstraint: ConstraintValues(valid="US-CA", invalid="99-999"), + SnakeCaseConstraint: ConstraintValues(valid="snake_case", invalid="HAS SPACES"), + StrippedConstraint: ConstraintValues(valid="clean", invalid=" has spaces "), + WikidataIdConstraint: ConstraintValues(valid="Q42", invalid="P999"), +} + + +# Curated valid/invalid pairs for fields whose only string constraint is a +# raw pydantic `Field(pattern=...)` (a `_PydanticGeneralMetadata`, not a +# schema constraint class -- so it has no `CONSTRAINT_VALUES` type key). +# Keyed by the anchor-normalized pattern that lands in the generated +# `check_pattern` descriptor's `args`, so both `base_row` and +# `invalid_value` look it up via `desc.args[0]`. An uncurated raw pattern +# fails loud on both sides rather than guessing a value. +# +# Generation-principle gap: this table is hand-maintained and keyed by the +# literal regex, so it drifts from the schema -- a renamed or retuned +# `Field(pattern=)` silently loses its entry until the next regeneration +# fails loud. The principled fix is to derive both sides from the regex +# itself (e.g. a matching/non-matching string generator), removing the +# hand-keyed table entirely. Out of scope here; tracked separately. +PATTERN_VALUES: dict[str, ConstraintValues] = { + # Sources.license_priority key (LicenseShortname): `^[A-Za-z0-9._+\-]+$`. + normalize_anchor(r"^[A-Za-z0-9._+\-]+$"): ConstraintValues( + valid="ODbL-1.0", invalid="bad license!" + ), +} + + +def curated_pattern_values(desc: ExpressionDescriptor) -> ConstraintValues | None: + """Curated valid/invalid pair for a raw-pattern `check_pattern` descriptor. + + The pattern key is the descriptor's first arg (the anchor-normalized + regex). Returns None when the pattern is not curated in `PATTERN_VALUES` + -- named constraints resolve via `CONSTRAINT_VALUES` instead, and an + uncurated raw pattern has no values. + """ + pattern = desc.args[0] if desc.args else None + if isinstance(pattern, str): + return PATTERN_VALUES.get(pattern) + return None + + +def uncurated_pattern_error(desc: ExpressionDescriptor, *, side: str) -> ValueError: + """Build the error for a `check_pattern` descriptor with no curated value. + + Raised symmetrically by `base_row` (valid side) and `invalid_value` + (invalid side) when a raw `Field(pattern=)` has no `PATTERN_VALUES` + entry: both name the table to update rather than guessing a value. + + Parameters + ---------- + desc + The uncurated `check_pattern` descriptor. + side + Which value could not be produced -- `"valid"` or `"invalid"`. + """ + return ValueError( + f"No {side} value defined for check_pattern with " + f"constraint_type={desc.constraint_type!r}, pattern={desc.args!r}. " + "Add an entry to CONSTRAINT_VALUES (named constraint) or " + "PATTERN_VALUES (raw pydantic pattern) in constraint_values.py." + ) + + +def valid_bound(desc: ExpressionDescriptor) -> object: + """Produce a value satisfying a bounds check for base row generation. + + Prefers inclusive boundaries: if `ge` is present it is already a valid + value; if `le` is present and `ge` is absent, `le` is valid. When only + exclusive bounds remain, a strictly-interior value is computed: midpoint + for both-exclusive, or a type-aware step away from a single bound. + + Parameters + ---------- + desc + A `check_bounds` descriptor with at least one bound kwarg. + + Returns + ------- + object + A value on the valid side of all bounds. Falls back to `0` when + no recognised bound key is present. + """ + kwargs = dict(desc.kwargs) + if "ge" in kwargs: + return kwargs["ge"] + if "le" in kwargs: + return kwargs["le"] + gt = kwargs.get("gt") + lt = kwargs.get("lt") + if gt is not None and lt is not None: + # Midpoint: integer midpoint for int bounds, float midpoint for float. + if isinstance(gt, float) or isinstance(lt, float): + return (float(gt) + float(lt)) / 2.0 # type: ignore[arg-type,operator] + mid = (gt + lt) // 2 # type: ignore[operator] + if not (gt < mid < lt): # type: ignore[operator] + raise ValueError( + f"No valid integer strictly between gt={gt!r} and lt={lt!r}" + ) + return mid + if gt is not None: + step: object = 1.0 if isinstance(gt, float) else 1 + return gt + step # type: ignore[operator] + if lt is not None: + step = 1.0 if isinstance(lt, float) else 1 + return lt - step # type: ignore[operator] + return 0 + + +def invalid_bound(desc: ExpressionDescriptor) -> object: + """Produce a value violating a bounds check for invalid-value generation. + + The `ge` / `le` branches return one below / above the bound. For + `ge=0` this returns `-1`, which violates the bound but would also + underflow an unsigned base type. No schema today combines `ge=0` with + an unsigned terminal -- if that ever changes, the caller will need to + consult the base type and pick a sentinel (e.g. a string or null) for + the violating value. + + Parameters + ---------- + desc + A `check_bounds` descriptor with at least one bound kwarg. + + Raises + ------ + ValueError + When no recognised bound key is found. + """ + kwargs = dict(desc.kwargs) + if "ge" in kwargs: + return kwargs["ge"] - 1 # type: ignore[operator] + if "gt" in kwargs: + return kwargs["gt"] + if "le" in kwargs: + return kwargs["le"] + 1 # type: ignore[operator] + if "lt" in kwargs: + return kwargs["lt"] + raise ValueError(f"No recognised bound key in kwargs: {kwargs!r}") diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/invalid_value.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/invalid_value.py new file mode 100644 index 000000000..e21812a84 --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/invalid_value.py @@ -0,0 +1,98 @@ +"""Generate constraint-violating values for the rendered conformance tests. + +`invalid_value` returns a concrete value that violates a given check. The +generated tests inject these into otherwise-valid rows to confirm that +each constraint produces the expected violation. +""" + +from __future__ import annotations + +from overture.schema.system.primitive.geom import GeometryType + +from ..constraint_dispatch import ExpressionDescriptor +from .constraint_values import ( + CONSTRAINT_VALUES, + curated_pattern_values, + invalid_bound, + uncurated_pattern_error, +) + +__all__ = ["invalid_value"] + +# Ordered candidates for the invalid geometry side (first not in allowed set wins) +_INVALID_GEOMETRY_CANDIDATES: tuple[tuple[GeometryType, str], ...] = ( + (GeometryType.POINT, "POINT (0 0)"), + (GeometryType.LINE_STRING, "LINESTRING (0 0, 1 1)"), + (GeometryType.GEOMETRY_COLLECTION, "GEOMETRYCOLLECTION EMPTY"), +) + + +# Direct lookup: check function name -> invalid value (no descriptor inspection). +# Reserved for checks with no associated constraint type (url/email, linear_range, +# bbox, required, enum, and min-length literals). +_INVALID_LITERALS: dict[str, object] = { + "check_required": None, + "check_enum": "__INVALID__", + "check_url_format": "not-a-url", + "check_url_length": "https://" + "x" * 2076, + "check_email": "not-an-email", + "check_array_min_length": [], + "check_string_min_length": "", + "check_linear_range_length": [0.5], + "check_linear_range_bounds": [1.5, 2.0], + "check_linear_range_order": [0.8, 0.2], + "check_bbox_completeness": {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0}, + "check_bbox_lat_ordering": {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0}, + "check_bbox_lat_range": {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0}, +} + + +def invalid_value(desc: ExpressionDescriptor) -> object: + """Return a Python value that violates `desc`'s check function. + + Parameters + ---------- + desc + The expression descriptor to produce an invalid value for. + + Raises + ------ + ValueError + For unrecognised check function names, unknown `constraint_type` + on `check_pattern` descriptors, or when all geometry candidates + are in the allowed set. + """ + fn = desc.function + # Constraint-type lookup precedes function-name lookup: any type present in + # CONSTRAINT_VALUES resolves via the table even when its check function also + # appears in _INVALID_LITERALS (e.g. check_stripped, check_json_pointer). + if desc.constraint_type in CONSTRAINT_VALUES: + return CONSTRAINT_VALUES[desc.constraint_type].invalid + if fn in _INVALID_LITERALS: + return _INVALID_LITERALS[fn] + if fn == "check_bounds": + return invalid_bound(desc) + if fn == "check_pattern": + if (curated := curated_pattern_values(desc)) is not None: + return curated.invalid + raise uncurated_pattern_error(desc, side="invalid") + if fn == "check_array_max_length": + max_len = int(desc.args[0]) # type: ignore[call-overload] + return [{}] * (max_len + 1) + if fn == "check_string_max_length": + max_len = int(desc.args[0]) # type: ignore[call-overload] + return "x" * (max_len + 1) + if fn == "check_geometry_type": + return _invalid_geometry(desc) + raise ValueError(f"No invalid value defined for check function: {fn!r}") + + +def _invalid_geometry(desc: ExpressionDescriptor) -> str: + allowed = set(desc.args) + for geom_type, wkt in _INVALID_GEOMETRY_CANDIDATES: + if geom_type not in allowed: + return wkt + raise ValueError( + f"All geometry candidates are in the allowed set: {allowed!r}. " + "Cannot produce an invalid geometry value." + ) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/scaffold.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/scaffold.py new file mode 100644 index 000000000..3d8d6554e --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/scaffold.py @@ -0,0 +1,341 @@ +"""Generate sparse path scaffolds for the rendered conformance tests. + +`generate_scaffold` builds a sparse dict that, when merged with a base +row, supplies the nested intermediates (optional structs, arrays) the +base row lacks but a check's field path requires. +`generate_model_scaffold` does the same for model-level constraints. +`leaf_list_depth` reports unaccounted-for list depth on a target field. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from overture.schema.system.field_path import ( + ArrayPath, + ArraySegment, + FieldPath, + FieldSegment, +) + +from ...extraction.field_walk import ( + has_array_layer, + list_depth, + terminal_model_ref, + terminal_union_ref, +) +from ...extraction.specs import FieldSpec, ModelSpec, RecordSpec +from ..check_ir import ( + Check, + ElementGuard, + ModelCheck, +) +from .base_row import ( + condition_overrides_for_present_field, + generate_base_row, + resolve_arm_spec, + value_for_field, +) + +__all__ = [ + "generate_model_scaffold", + "generate_scaffold", + "leaf_list_depth", +] + +# Sentinel for "no leaf override": the terminal field keeps its synthesized +# value. A `None` / `""` leaf override is meaningful, so it cannot be the +# default. +_UNSET: object = object() + + +def _nest_leaf_value(value: object, field_spec: FieldSpec) -> object: + """Wrap a scalar leaf override to the field's list nesting depth. + + `value_for_field` returns a list for a list-typed field, so a bare scalar + override (e.g. a literal alternative) is wrapped to the same depth: `[v]` + for `list[T]`, `[[v]]` for `list[list[T]]`, and `v` for a scalar field. + """ + for _ in range(list_depth(field_spec.shape)): + value = [value] + return value + + +@dataclass(frozen=True, slots=True) +class _ElementDiscriminator: + """Discriminator value to seed at one nesting depth of the scaffold.""" + + field: str + value: str + depth: int + + +def _find_field_spec(fields: list[FieldSpec], name: str) -> FieldSpec | None: + """Find a FieldSpec by name in a list.""" + for f in fields: + if f.name == name: + return f + return None + + +def leaf_list_depth(field_path: FieldPath, spec: ModelSpec) -> int: + """Return the unaccounted-for list depth of the leaf field. + + Walks the spec's field tree along *field_path* and returns the + leaf's `list_depth(shape)` minus any `iter_count` on the terminal + path segment. Paths whose terminal segment is itself an array + target the array's elements, so the mutation already operates one + level deep. Returns 0 when *field_path* is empty or when any + segment fails to resolve against *spec* (e.g. union arms that + don't share the path's intermediate fields). + """ + segments = field_path.segments + if not segments: + return 0 + fields = list(spec.fields) + for seg in segments[:-1]: + field = _find_field_spec(fields, seg.name) + if field is None: + return 0 + model_ref = terminal_model_ref(field.shape) + if model_ref is None: + return 0 + fields = model_ref.model.fields + leaf_seg = segments[-1] + leaf = _find_field_spec(fields, leaf_seg.name) + if leaf is None: + return 0 + terminal_iter = leaf_seg.iter_count if isinstance(leaf_seg, ArraySegment) else 0 + return max(0, list_depth(leaf.shape) - terminal_iter) + + +def _child_container_spec( + field_spec: FieldSpec, discriminator_value: object | None +) -> RecordSpec | None: + """Resolve the model a path field descends into. + + Returns the field's terminal `ModelRef` model, or -- for a discriminated + union -- the member arm the `discriminator_value` selects (the widest + member when the check is not arm-gated). `None` when the field has neither + a model nor a union terminal. + """ + model_ref = terminal_model_ref(field_spec.shape) + if model_ref is not None: + return model_ref.model + union_ref = terminal_union_ref(field_spec.shape) + if union_ref is not None: + return resolve_arm_spec(union_ref.union, discriminator_value) + return None + + +def _walk_to_target( + segments: tuple[FieldSegment, ...], + fields: list[FieldSpec], + spec_name: str, + *, + discriminator: _ElementDiscriminator | None, + current_depth: int = 0, + leaf_value: object = _UNSET, +) -> dict[str, Any]: + """Recursively build a constraint-satisfying scaffold along the path. + + Each container model on the path is built as a valid base row + (`generate_base_row` -- required fields populated and model constraints + such as `require_any_of` satisfied), then the on-path child overrides its + field. A discriminated-union element resolves to the arm the seeded + discriminator selects (or the widest member when the check is not + arm-gated), so the element is a valid instance of a concrete arm rather + than an untagged `{}`. + + Accepts any `FieldSegment`: struct steps recurse, an `ArraySegment` + wraps its inner value in lists, and a trailing `MapSegment` resolves + via `value_for_field` (which populates the map with a valid entry), + so a `MapPath` target scaffolds the same way as a struct terminal. + + `leaf_value`, when set, replaces the synthesized value at the terminal + field -- used to seed a specific valid value (e.g. a literal alternative) + at the check's target. + """ + if not segments: + return {} + + seg = segments[0] + remaining = segments[1:] + field_spec = _find_field_spec(fields, seg.name) + + # A path segment that resolves to no field, or that tries to descend into a + # non-container, would leave the scaffold short of its target -- the + # `::valid` row would then assert nothing (the vacuous-valid-row bug this + # generator exists to prevent). Fail loud at generation time instead. + if field_spec is None: + raise ValueError( + f"scaffold path segment {seg.name!r} matches no field " + f"(available: {sorted(f.name for f in fields)})" + ) + + inner: Any + if remaining: + discriminator_value = ( + discriminator.value + if discriminator is not None and current_depth == discriminator.depth + else None + ) + child_spec = _child_container_spec(field_spec, discriminator_value) + if child_spec is None: + raise ValueError( + f"scaffold cannot descend into non-container field {seg.name!r} " + f"with path remaining {[s.name for s in remaining]!r}" + ) + recursed = _walk_to_target( + remaining, + child_spec.fields, + spec_name, + discriminator=discriminator, + current_depth=current_depth + 1, + leaf_value=leaf_value, + ) + inner = {**generate_base_row(child_spec), **recursed} + elif leaf_value is not _UNSET: + inner = _nest_leaf_value(leaf_value, field_spec) + else: + inner = value_for_field(field_spec, spec_name) + + if ( + isinstance(inner, dict) + and discriminator is not None + and current_depth == discriminator.depth + ): + inner[discriminator.field] = discriminator.value + + # When the terminal segment is an array and the field itself is a list, + # `value_for_field` already wrapped the value -- skip extra wrapping. + if isinstance(seg, ArraySegment): + if not remaining and has_array_layer(field_spec.shape): + return {seg.name: inner} + # A single-level array (iter_count == 1) gets a constraint-valid list; + # nested `list[list[...]]` levels (iter_count > 1) carry no min_length>1 + # or uniqueness constraint in any current schema, so minimal nesting + # suffices. Add per-level constraint handling here if one ever does -- + # the row would otherwise be short on the unmutated `::valid` row. + if seg.iter_count == 1: + return {seg.name: _array_with_target(inner, field_spec, spec_name)} + wrapped: Any = inner + for _ in range(seg.iter_count): + wrapped = [wrapped] + return {seg.name: wrapped} + if remaining and has_array_layer(field_spec.shape): + return {seg.name: _array_with_target(inner, field_spec, spec_name)} + return {seg.name: inner} + + +def _array_with_target( + target_element: object, field_spec: FieldSpec, spec_name: str +) -> list[Any]: + """Return a constraint-valid single-level list holding the target element. + + `value_for_field` builds a list that satisfies the field's array + constraints (min length, unique items); the target-reaching element + replaces the first slot. A min_length>1 or uniqueness constraint then + holds on the unmutated `::valid` row -- a bare `[target_element]` would + leave the row short or, after `deep_merge` replaces the base row's list, + drop the elements that satisfied the constraint. + """ + full = value_for_field(field_spec, spec_name) + if isinstance(full, list) and full: + full[0] = target_element + return full + return [target_element] + + +def _element_discriminator(check: Check) -> _ElementDiscriminator | None: + """Return the element-level discriminator for a Check, or None. + + Bundles the discriminator field, the value to seed, and the depth at + which to seed it (the innermost array segment in the target path). + The check_ir invariant is that nested-union gating composes at most + one `ElementGuard` per Check; more than one would mean the gate + composition rule changed without updating the scaffold, so raise to + surface the gap rather than silently dropping guards. + """ + element_guards = [g for g in check.guards if isinstance(g, ElementGuard)] + if len(element_guards) > 1: + raise NotImplementedError( + f"Check carries {len(element_guards)} ElementGuards " + f"({element_guards!r}); the scaffold only seeds one. Update " + "the scaffold builder when the gate composition rule changes." + ) + if not element_guards or not element_guards[0].values: + return None + guard = element_guards[0] + segments = check.target.segments + for i in range(len(segments) - 1, -1, -1): + if isinstance(segments[i], ArraySegment): + return _ElementDiscriminator( + field=guard.discriminator, value=guard.values[0], depth=i + ) + return None + + +def generate_scaffold( + check: Check, spec: ModelSpec, *, leaf_value: object = _UNSET +) -> dict[str, Any]: + """Build a sparse dict from null to the target field of a Check. + + `leaf_value`, when set, seeds that value at the target instead of the + field's synthesized value -- used to place a known-valid value (e.g. a + literal alternative) at the check's target for the `::valid` row. + """ + segments = check.target.segments + if not segments: + return {} + + if len(segments) == 1: + seg0 = segments[0] + field_spec = _find_field_spec(spec.fields, seg0.name) + if field_spec is None: + return {} + # A `forbid_if` the base row triggers forbids this field; disable the + # condition so the field can be set without invalidating the row. + overrides = condition_overrides_for_present_field(spec, seg0.name) + if leaf_value is not _UNSET: + return {**overrides, seg0.name: _nest_leaf_value(leaf_value, field_spec)} + if field_spec.is_required: + return {} + return {**overrides, seg0.name: value_for_field(field_spec, spec.name)} + + return _walk_to_target( + segments, + spec.fields, + spec.name, + discriminator=_element_discriminator(check), + leaf_value=leaf_value, + ) + + +def generate_model_scaffold(check: ModelCheck, spec: ModelSpec) -> dict[str, Any]: + """Build a constraint-satisfying scaffold for a model-level check. + + Two target shapes need no scaffold and return `{}`: + + - a `ScalarPath` target -- a top-level model constraint, whose fields + live at the row root; + - a `MapPath` target -- a `dict[K, Model]` value-model constraint. The + mutation (`map_path=`) owns map navigation: it corrupts the base row's + single map entry in place, or stubs one when the map is absent. Unlike + an array, a dict scaffold can't replace a base-row map entry under + `deep_merge`'s recursive dict merge, so there is nothing to add here. + + An `ArrayPath` walks the path with `_walk_to_target`: every model on the + way -- including the constrained model at the leaf -- is built as a valid + base row, so the constraint under test (e.g. a scope's `require_any_of`) + is satisfied on the unmutated `::valid` row and the only violation is the + one the mutation introduces. + """ + match check.target: + case ArrayPath() as target: + return _walk_to_target( + target.segments, spec.fields, spec.name, discriminator=None + ) + case _: + return {} diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_renderer.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_renderer.py new file mode 100644 index 000000000..6d245a046 --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_renderer.py @@ -0,0 +1,540 @@ +"""Render Check / ModelCheck IR into generated conformance test modules.""" + +from __future__ import annotations + +from typing import Any, NamedTuple + +from typing_extensions import assert_never + +from overture.schema.system.field_path import ArrayPath, MapPath, MapProjection + +from ..extraction.field import FieldShape, Primitive +from ..extraction.field_walk import has_array_layer, terminal_of +from ..extraction.specs import ModelSpec +from ..extraction.type_registry import primitive_spark_category +from ._primitive_fill import PRIMITIVE_FILL_TABLE +from ._render_common import ( + disambiguate, + field_check_rows, + jinja_env, + model_check_rows, + py_literal, + schema_const_name, +) +from .check_ir import ( + Check, + ColumnGuard, + ModelCheck, +) +from .constraint_dispatch import ( + ExpressionDescriptor, + ForbidIf, + MinFieldsSet, + ModelConstraintDescriptor, + RadioGroup, + RequireAnyOf, + RequireAnyTrue, + RequireIf, + model_constraint_function, + model_mutation_function, + parse_field_eq, + require_bool_field_eq, +) +from .test_data.invalid_value import invalid_value +from .test_data.scaffold import ( + generate_model_scaffold, + generate_scaffold, + leaf_list_depth, +) + +__all__ = ["render_test_module"] + + +def _check_belongs_to_arm(check: Check, arm: str) -> bool: + """Return True when a Check applies to a given union arm. + + The outermost union's discriminator surfaces as `ColumnGuard`s; inner + unions use `ElementGuard`s on a different discriminator field and are + irrelevant to arm filtering. A check belongs to *arm* when every + `ColumnGuard` admits it (guards are AND-composed). + """ + column_guards = [g for g in check.guards if isinstance(g, ColumnGuard)] + return all(arm in g.values for g in column_guards) + + +def _model_check_belongs_to_arm(check: ModelCheck, arm: str) -> bool: + """Return True when a ModelCheck applies to a given union arm. + + `ModelCheck.arm` is `None` for union-level constraints (which apply + regardless of discriminator) and set to a discriminator value for + constraints contributed by one specific member class. + """ + return check.arm is None or check.arm == arm + + +def render_test_module( + model_name: str, + field_checks: list[Check], + model_checks: list[ModelCheck], + *, + expression_import: str, + support_prefix: str, + base_row_sparse: dict[str, Any] | None = None, + base_row_populated: dict[str, Any] | None = None, + arm: str | None = None, + spec: ModelSpec | None = None, +) -> str: + """Render a complete pytest test file for a model's validation checks. + + Arm filtering uses two complementary signals. A field check's + `ColumnGuard`s identify the arms it belongs to. A model check's `arm` + attribute is set for member-specific constraints and `None` for + union-level constraints (which apply to every arm). + + Both label-collision passes run over the *unfiltered* check lists so + they agree with the expression module, which `renderer` emits once + across every arm. Each scenario builder takes `arm` and drops rows + that fall outside it after their suffixes are assigned; computing a + suffix over an arm subset would let it hide a collision the shared + module still carries, producing an `expected_field` the module never + emits. + """ + model_scenarios, used_mutation_fns = _render_model_scenarios( + model_name, model_checks, spec, arm + ) + field_scenarios, field_helpers = _render_field_check_scenarios( + model_name, field_checks, spec, arm + ) + used_mutation_fns |= field_helpers - {"set_at_path"} + + sparse_repr = py_literal(base_row_sparse) if base_row_sparse is not None else "{}" + populated_repr = ( + py_literal(base_row_populated) if base_row_populated is not None else "{}" + ) + + all_scenarios = field_scenarios + model_scenarios + + template = jinja_env().get_template("test_module.py.jinja2") + return template.render( + model_name=model_name, + schema_name=schema_const_name(model_name), + mutation_imports=sorted(used_mutation_fns), + needs_set_at_path="set_at_path" in field_helpers, + base_row_sparse=sparse_repr, + base_row_populated=populated_repr, + scenarios=all_scenarios, + expression_import=expression_import, + support_prefix=support_prefix, + ) + + +def _scenario_entry( + *, + scenario_id: str, + scaffold: dict[str, Any], + mutate_expr: str, + expected_field: str, + expected_check: str, + valid_scaffold: dict[str, Any] | None = None, +) -> list[tuple[str, str]]: + """Build a rendered Scenario kwargs list for the test_module template. + + `valid_scaffold` is emitted only when set, so scenarios without one keep + the dataclass default (a vacuous base-row copy for the `::valid` row). + """ + entry = [ + ("id", py_literal(scenario_id)), + ("scaffold", py_literal(scaffold)), + ("mutate", mutate_expr), + ("expected_field", py_literal(expected_field)), + ("expected_check", py_literal(expected_check)), + ] + if valid_scaffold is not None: + entry.append(("valid_scaffold", py_literal(valid_scaffold))) + return entry + + +class _MutateExpr(NamedTuple): + """One rendered `mutate=` expression and the helper it imports. + + `helper` is `None` when the expression is a literal `set_at_path` + call (the default), and otherwise names a `mutate_*` helper from + `tests/_support/mutations.py` to import. + """ + + expr: str + helper: str | None + + +def _field_mutate_expr( + check: Check, desc: ExpressionDescriptor, spec: ModelSpec | None +) -> _MutateExpr: + """Render the `mutate=` expression for one field-check descriptor. + + A `MapPath` target corrupts the map's single valid entry via + `mutate_map_key` / `mutate_map_value`; `check_struct_unique` calls + `mutate_unique_items` at the target path; every other descriptor + injects a constraint-violating literal via `set_at_path`. + """ + if isinstance(check.target, MapPath): + helper = ( + "mutate_map_key" + if check.target.projection is MapProjection.KEY + else "mutate_map_value" + ) + col_repr = py_literal(check.target.map_column) + iv_repr = py_literal(invalid_value(desc)) + return _MutateExpr(f"lambda row: {helper}(row, {col_repr}, {iv_repr})", helper) + target_repr = py_literal(str(check.target)) + if desc.function == "check_struct_unique": + return _MutateExpr( + f"lambda row: mutate_unique_items(row, {target_repr})", + "mutate_unique_items", + ) + iv_val = _wrap_for_list_leaf(invalid_value(desc), check, spec) + return _MutateExpr(f"set_at_path({target_repr}, {py_literal(iv_val)})", None) + + +def _render_field_check_scenarios( + model_name: str, + field_checks: list[Check], + spec: ModelSpec | None, + arm: str | None, +) -> tuple[list[list[tuple[str, str]]], set[str]]: + """Render Scenario entries for field-level checks. + + Returns the entries and the set of mutation helper names referenced + by them, mirroring `_render_model_scenarios`. `field_check_rows` + assigns collision suffixes over the unfiltered list; this drops rows + outside `arm` afterward so per-arm modules carry the labels the shared + expression module emits. Pass `None` to include all arms. + """ + rows = [ + row + for row in field_check_rows(field_checks) + if arm is None or _check_belongs_to_arm(row.check, arm) + ] + scenario_ids = disambiguate( + [f"{model_name}::{row.label}:{row.name}" for row in rows] + ) + + entries: list[list[tuple[str, str]]] = [] + used_helpers: set[str] = set() + for row, scenario_id in zip(rows, scenario_ids, strict=True): + desc = row.check.descriptors[row.descriptor_idx] + scaffold = generate_scaffold(row.check, spec) if spec is not None else {} + # For an `X | Literal[c]` field, seed the literal alternative at the + # target so the `::valid` row proves the check accepts it. + valid_scaffold: dict[str, Any] | None = None + if desc.allow_literals and spec is not None: + # generate_scaffold shapes the bare literal to the field's list + # nesting, so pass it unwrapped. + valid_scaffold = generate_scaffold( + row.check, spec, leaf_value=desc.allow_literals[0] + ) + try: + mutate = _field_mutate_expr(row.check, desc, spec) + except ValueError as exc: + raise ValueError( + f"Cannot render mutate expression for {scenario_id}: {exc}" + ) from exc + used_helpers.add(mutate.helper or "set_at_path") + entries.append( + _scenario_entry( + scenario_id=scenario_id, + scaffold=scaffold, + mutate_expr=mutate.expr, + expected_field=row.label, + expected_check=row.name, + valid_scaffold=valid_scaffold, + ) + ) + + return entries, used_helpers + + +def _checks_array_element(check: Check) -> bool: + """True when the check fires on each element of an `ArrayPath` directly. + + The check target ends at the array (`leaf=()`), so the mutation + replaces an array element rather than a struct field on one. For + these checks, a `None` invalid value still needs list wrapping; for + nested struct fields, `None` already sits at the right level. + """ + return isinstance(check.target, ArrayPath) and not check.target.leaf + + +def _wrap_for_list_leaf( + value: object, + check: Check, + spec: ModelSpec | None, +) -> object: + """Wrap a scalar invalid value to match the field's list nesting depth.""" + if spec is None or isinstance(value, list): + return value + if value is None and not _checks_array_element(check): + return value + depth = leaf_list_depth(check.target, spec) + for _ in range(depth): + value = [value] + return value + + +def _render_model_scenarios( + model_name: str, + model_checks: list[ModelCheck], + spec: ModelSpec | None, + arm: str | None, +) -> tuple[list[list[tuple[str, str]]], set[str]]: + """Render Scenario entries for model-level checks. + + Returns the entries and the set of mutation helper names referenced + by them, so the caller can scope the test module's imports. + `model_check_rows` assigns collision suffixes over the unfiltered + list; this drops rows outside `arm` afterward so per-arm modules carry + the labels the shared expression module emits. Pass `None` to include + all arms. + + The scenario id's trailing index counts surviving rows within the arm + (`enumerate` after the filter), not the row's position in the + unfiltered list -- it is a test-internal disambiguator with no + cross-module contract, kept contiguous per arm. + """ + entries: list[list[tuple[str, str]]] = [] + used_mutation_fns: set[str] = set() + + rows = [ + row + for row in model_check_rows(model_checks) + if arm is None or _model_check_belongs_to_arm(row.check, arm) + ] + for scenario_idx, row in enumerate(rows): + mc = row.check + desc = mc.descriptor + mutation_fn = model_mutation_function(desc) + scenario_id = f"{model_name}::model:{row.name}:{scenario_idx}" + scaffold = generate_model_scaffold(mc, spec) if spec is not None else {} + + try: + call = _render_mutation_call(mutation_fn, desc, mc) + except ValueError as exc: + raise ValueError( + f"Cannot render mutation call for {scenario_id}: {exc}" + ) from exc + mutate_expr = f"lambda row: {call}" + used_mutation_fns.add(mutation_fn) + entries.append( + _scenario_entry( + scenario_id=scenario_id, + scaffold=scaffold, + mutate_expr=mutate_expr, + expected_field=row.label, + expected_check=row.name, + ) + ) + + return entries, used_mutation_fns + + +def _render_mutation_call( + mutation_fn: str, + desc: ModelConstraintDescriptor, + check: ModelCheck, +) -> str: + """Render a model mutation helper function call.""" + fields_repr = py_literal(list(desc.field_names)) + + match desc: + case RequireAnyTrue(): + # Carries `conditions`, not `field_names`: the mutation disables + # every condition via a per-field `{field: value}` dict rather than + # the shared field-name list the other descriptors pass. + return _render_require_any_true_mutation_call(mutation_fn, desc) + case RequireIf() | ForbidIf(): + return _render_conditional_mutation_call( + mutation_fn, desc, check, fields_repr + ) + case RadioGroup(): + if isinstance(check.target, (ArrayPath, MapPath)): + raise ValueError( + "mutate_radio_group does not accept array_path/map_path " + f"(target={check.target!r})" + ) + return f"{mutation_fn}(row, {fields_repr})" + case RequireAnyOf() | MinFieldsSet(): + parts = _iter_kwargs_leaf(check, mutation_fn) + suffix = ", " + ", ".join(parts) if parts else "" + return f"{mutation_fn}(row, {fields_repr}{suffix})" + assert_never(desc) + + +def _render_conditional_mutation_call( + mutation_fn: str, + desc: RequireIf | ForbidIf, + check: ModelCheck, + fields_repr: str, +) -> str: + """Render a mutate_require_if or mutate_forbid_if call.""" + parsed = parse_field_eq(desc.condition) + fn = model_constraint_function(desc) + if parsed is None: + raise ValueError( + f"{fn} condition {desc.condition!r} is not a " + "FieldEqCondition or Not(FieldEqCondition); cannot render " + f"{mutation_fn} call" + ) + fill = _render_fill_values(desc) if isinstance(desc, ForbidIf) else None + kwarg_parts: list[str] = [] + if parsed.negated: + kwarg_parts.append("negate=True") + if fill: + kwarg_parts.append(f"fill_values={fill}") + kwarg_parts.extend(_iter_kwargs_inner(check, mutation_fn)) + suffix = ", " + ", ".join(kwarg_parts) if kwarg_parts else "" + return ( + f"{mutation_fn}(row, {fields_repr}, " + f"{py_literal(parsed.field_name)}, {py_literal(parsed.value)}{suffix})" + ) + + +def _render_require_any_true_mutation_call( + mutation_fn: str, desc: RequireAnyTrue +) -> str: + """Render a `mutate_require_any_true` call. + + Passes a `{field: disabling_value}` dict that makes every condition false, + so the invalid row violates `require_any_true` and nothing else. Conditions + are positive boolean equalities (`require_bool_field_eq`), so each field's + disabling value is the negation of the boolean the condition tests for. + """ + parsed = [require_bool_field_eq(c) for c in desc.conditions] + items = ", ".join( + f"{py_literal(p.field_name)}: {py_literal(not p.value)}" for p in parsed + ) + return f"{mutation_fn}(row, {{{items}}})" + + +def _fill_value_literal(shape: FieldShape) -> str: + """Return a Python source literal for a type-appropriate non-null fill value.""" + if has_array_layer(shape): + return "[{}]" + terminal = terminal_of(shape) + if isinstance(terminal, Primitive): + category = primitive_spark_category(terminal.base_type) + if category in PRIMITIVE_FILL_TABLE: + return PRIMITIVE_FILL_TABLE[category][0] + raise ValueError(f"unhandled Primitive base_type: {terminal.base_type!r}") + return "{}" + + +def _render_fill_values(desc: ForbidIf) -> str | None: + """Render a `fill_values` dict literal for non-string ForbidIf targets.""" + if not desc.field_shapes: + return None + items = [ + f"{py_literal(name)}: {_fill_value_literal(shape)}" + for name, shape in desc.field_shapes + ] + return "{" + ", ".join(items) + "}" + + +def _map_kwargs(target: MapPath, mutation_fn: str, *, allow_leaf: bool) -> list[str]: + """Mutation kwargs for a `dict[K, Model]` value-model constraint. + + Emits `map_path=...` (the map column) and, when `allow_leaf`, an + optional single-segment `struct_path=...` for a sub-model reached + through one struct field inside the value model -- the map analogue of + `_iter_kwargs_leaf`'s array `struct_path`. A KEY projection is + unrepresentable (a model can't be a dict key) and raises; a multi-segment + leaf, or any leaf when `allow_leaf` is False, raises too. + """ + if target.projection is not MapProjection.VALUE: + raise ValueError( + f"{mutation_fn} cannot target a map key (target={target!r}); a " + "model-level constraint on a map key is not representable as a row" + ) + kwargs = [f'map_path="{target.map_column}"'] + leaf = target.leaf + if leaf: + if not allow_leaf: + raise ValueError( + f"{mutation_fn} does not accept a map-value leaf (leaf={leaf!r})" + ) + if len(leaf) > 1: + raise ValueError( + f"multi-segment map-value leaf {leaf!r} not supported by " + f"{mutation_fn} (struct_path must be a single segment)" + ) + kwargs.append(f'struct_path="{leaf[0]}"') + return kwargs + + +def _iter_kwargs_leaf(check: ModelCheck, mutation_fn: str) -> list[str]: + """Container kwargs for mutations accepting `struct_path` (a trailing leaf). + + For an `ArrayPath`, yields `array_path=...` and optionally + `struct_path=...`; inner array iteration is rejected -- these mutations + consume only the outermost array level. For a `MapPath` (a + `dict[K, Model]` value-model constraint), delegates to `_map_kwargs`, + which yields `map_path=...` and an optional single-segment `struct_path`. + """ + if isinstance(check.target, MapPath): + return _map_kwargs(check.target, mutation_fn, allow_leaf=True) + if not isinstance(check.target, ArrayPath): + return [] + inner_struct_paths = check.target.iter_struct_paths + leaf_path = check.target.leaf + + if inner_struct_paths: + raise ValueError( + f"{mutation_fn} does not accept inner_array_path " + f"(inner struct paths={inner_struct_paths!r})" + ) + + kwargs = [f'array_path="{check.target.column_path}"'] + if leaf_path: + if len(leaf_path) > 1: + raise ValueError( + f"multi-segment leaf_path {leaf_path!r} not supported by " + f"{mutation_fn} (struct_path must be a single segment)" + ) + kwargs.append(f'struct_path="{leaf_path[0]}"') + return kwargs + + +def _iter_kwargs_inner(check: ModelCheck, mutation_fn: str) -> list[str]: + """Container kwargs for mutations accepting `inner_array_path`. + + For an `ArrayPath`, yields `array_path=...` and optionally + `inner_array_path=...`; a trailing leaf path is rejected -- these + mutations target an inner array directly, not a struct field on its + elements. For a `MapPath`, delegates to `_map_kwargs` (no leaf: a map + value has no inner array layer to address). + """ + if isinstance(check.target, MapPath): + return _map_kwargs(check.target, mutation_fn, allow_leaf=False) + if not isinstance(check.target, ArrayPath): + return [] + inner_struct_paths = check.target.iter_struct_paths + leaf_path = check.target.leaf + + if leaf_path: + raise ValueError( + f"{mutation_fn} does not accept struct_path (leaf_path={leaf_path!r})" + ) + + kwargs = [f'array_path="{check.target.column_path}"'] + if inner_struct_paths: + if len(inner_struct_paths) > 1: + raise ValueError( + f"multi-level inner struct paths {inner_struct_paths!r} not supported by " + f"{mutation_fn} (inner_array_path consumes one iteration)" + ) + if not inner_struct_paths[0]: + raise ValueError( + f"empty inner struct path not supported by {mutation_fn} " + f"(target={check.target!r}); nested-iteration arrays without " + f"intermediate struct fields cannot be addressed via inner_array_path" + ) + kwargs.append(f'inner_array_path="{".".join(inner_struct_paths[0])}"') + return kwargs diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/spec_discovery.py b/packages/overture-schema-codegen/src/overture/schema/codegen/spec_discovery.py new file mode 100644 index 000000000..7bcf5f54c --- /dev/null +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/spec_discovery.py @@ -0,0 +1,44 @@ +"""Bridge discovered models to extracted specs. + +`discover_models` yields `(ModelKey, entry)` pairs where each entry is either +a concrete Pydantic model class or a discriminated-union type alias. This +module turns one such pair into its `ModelSpec`, applying the partition layout +and entry point uniformly so every call site shares the same extraction. It +sits at the orchestration tier (alongside `cli`), importing downward into +extraction and layout. +""" + +from __future__ import annotations + +from overture.schema.system.discovery import ModelKey + +from .extraction.model_extraction import extract_model +from .extraction.specs import ( + ModelSpec, + is_model_class, + is_union_alias, + partitions_from_tags, +) +from .extraction.union_extraction import extract_union +from .layout.module_layout import entry_point_class + +__all__ = ["extract_model_spec"] + + +def extract_model_spec(key: ModelKey, entry: object) -> ModelSpec | None: + """Extract the `ModelSpec` for one discovered `(key, entry)` pair. + + Returns None when `entry` is neither a concrete model class nor a union + alias, so callers can skip non-model entries with a single check. + """ + partitions = partitions_from_tags(key.tags) + if is_model_class(entry): + return extract_model(entry, entry_point=key.entry_point, partitions=partitions) + if is_union_alias(entry): + return extract_union( + entry_point_class(key.entry_point), + entry, + entry_point=key.entry_point, + partitions=partitions, + ) + return None diff --git a/packages/overture-schema-codegen/tests/codegen_test_support.py b/packages/overture-schema-codegen/tests/codegen_test_support.py index 2a18faf13..c30bffebb 100644 --- a/packages/overture-schema-codegen/tests/codegen_test_support.py +++ b/packages/overture-schema-codegen/tests/codegen_test_support.py @@ -12,6 +12,9 @@ from typing import Annotated, Generic, Literal, NewType, TypeVar import pytest +from annotated_types import MinLen +from overture.schema.codegen.extraction.field import LiteralScalar, Primitive +from overture.schema.codegen.extraction.field_walk import terminal_of from overture.schema.codegen.extraction.model_extraction import extract_model from overture.schema.codegen.extraction.pydantic_extraction import extract_pydantic_type from overture.schema.codegen.extraction.specs import ( @@ -19,12 +22,17 @@ EnumMemberSpec, EnumSpec, FieldSpec, + MemberSpec, ModelSpec, + RecordSpec, TypeIdentity, UnionSpec, is_model_class, + is_union_alias, ) -from overture.schema.codegen.extraction.type_analyzer import TypeInfo, TypeKind +from overture.schema.codegen.extraction.union_extraction import extract_union +from overture.schema.codegen.layout.module_layout import entry_point_class +from overture.schema.codegen.spec_discovery import extract_model_spec from overture.schema.system.discovery import ( TagSelector, discover_models, @@ -33,7 +41,12 @@ from overture.schema.system.discovery.tag import get_values_for_key from overture.schema.system.doc import DocumentedEnum from overture.schema.system.field_constraint import UniqueItemsConstraint -from overture.schema.system.model_constraint import require_any_of +from overture.schema.system.model_constraint import ( + FieldEqCondition, + radio_group, + require_any_of, + require_any_true, +) from overture.schema.system.primitive import ( Geometry, GeometryType, @@ -45,7 +58,7 @@ from overture.schema.system.string import HexColor, LanguageTag, StrippedString from pydantic import BaseModel, EmailStr, Field, HttpUrl -STR_TYPE = TypeInfo(base_type="str", kind=TypeKind.PRIMITIVE) +STR_TYPE = Primitive(base_type="str") ThemeT = TypeVar("ThemeT") TypeT = TypeVar("TypeT") @@ -210,6 +223,20 @@ class FeatureWithUrl(FeatureBase[Literal["test"], Literal["linked"]]): emails: list[EmailStr] | None = None +class DatasetEntry(BaseModel): + """A dataset with required URL fields.""" + + name: str = Field(description="Dataset name") + url: HttpUrl + download_urls: list[HttpUrl] | None = None + + +class FeatureWithRequiredUrl(FeatureBase[Literal["test"], Literal["urlreq"]]): + """A feature with required URL fields at multiple nesting levels.""" + + datasets: list[DatasetEntry] + + HTTP_URL_SPEC = extract_pydantic_type(HttpUrl) EMAIL_STR_SPEC = extract_pydantic_type(EmailStr) @@ -243,6 +270,49 @@ class WaterSegment(SegmentBase): ] +class ShortNamesSegment(SegmentBase): + """Segment variant whose `aliases` requires at least one entry.""" + + subtype: Literal["short"] + aliases: Annotated[list[str], Field(min_length=1)] | None = None + + +class LongNamesSegment(SegmentBase): + """Segment variant whose `aliases` requires at least five entries.""" + + subtype: Literal["long"] + aliases: Annotated[list[str], Field(min_length=5)] | None = None + + +TestSegmentDivergingConstraints = Annotated[ + ShortNamesSegment | LongNamesSegment, + Field(description="Union whose members declare diverging field constraints"), +] + + +class VehicleKind(str, Enum): + """Vehicle classification.""" + + CAR = "car" + BIKE = "bike" + + +class CarVariant(SegmentBase): + subtype: Literal[VehicleKind.CAR] + doors: int | None = None + + +class BikeVariant(SegmentBase): + subtype: Literal[VehicleKind.BIKE] + has_basket: bool | None = None + + +TestEnumDiscriminatorUnion = Annotated[ + CarVariant | BikeVariant, + Field(description="Union with enum-valued discriminator", discriminator="subtype"), +] + + class ContactInfo(BaseModel): """Contact information for a venue.""" @@ -273,16 +343,23 @@ def make_union_spec( common_base: type[BaseModel] | None = None, entry_point: str | None = None, ) -> UnionSpec: - """Build a UnionSpec with sensible defaults for tests.""" + """Build a UnionSpec with sensible defaults for tests. + + `member_specs` is derived from `members` via `extract_model`, matching + what `extract_union` produces, so specs built here behave the same + through `_model_checks_for_union` and the base-row generators. + """ + members = members or [] return UnionSpec( name=name, description=description, annotated_fields=annotated_fields or [], - members=members or [], + members=members, discriminator_field=None, discriminator_mapping=None, source_annotation=source_annotation, common_base=common_base or BaseModel, + member_specs=[MemberSpec(m, extract_model(m)) for m in members], entry_point=entry_point, ) @@ -297,8 +374,8 @@ def find_model_class(name: str, models: dict[object, object]) -> type[BaseModel] return match -def find_field(spec: ModelSpec, name: str) -> FieldSpec: - """Find a field by name in a ModelSpec, raising if missing.""" +def find_field(spec: RecordSpec, name: str) -> FieldSpec: + """Find a field by name in a RecordSpec, raising if missing.""" return next(f for f in spec.fields if f.name == name) @@ -329,29 +406,115 @@ def has_name(mapping: Mapping[TypeIdentity, object], name: str) -> bool: def assert_literal_field( - spec: ModelSpec, field_name: str, expected_value: object + spec: RecordSpec, field_name: str, expected_value: object ) -> None: """Assert a field is a single-value Literal with the expected value.""" field = find_field(spec, field_name) - assert field.type_info.kind == TypeKind.LITERAL - assert field.type_info.literal_values == (expected_value,) + terminal = terminal_of(field.shape) + assert isinstance(terminal, LiteralScalar) + assert terminal.values == (expected_value,) def flat_specs_from_discovery( theme: str | None = None, -) -> list[ModelSpec]: - """Build a flat list of ModelSpecs from discovery, with entry_point set.""" +) -> list[RecordSpec]: + """Build a flat list of RecordSpecs from discovery, with entry_point set.""" models = discover_models() if theme: models = filter_models( models, TagSelector(include_any=(f"overture:theme={theme}",)) ) - result = [] - for key, cls in models.items(): - if not is_model_class(cls): - continue - result.append(extract_model(cls, entry_point=key.entry_point)) - return result + return [ + spec + for key, cls in models.items() + if isinstance(spec := extract_model_spec(key, cls), RecordSpec) + ] + + +class TaggedVariantA(SegmentBase): + """Segment variant with a unique-items tags field.""" + + subtype: Literal["tagged_a"] + tags: Annotated[list[str], UniqueItemsConstraint()] | None = None + + +class TaggedVariantB(SegmentBase): + """Segment variant with a unique-items tags field (distinct instance, same constraint).""" + + subtype: Literal["tagged_b"] + tags: Annotated[list[str], UniqueItemsConstraint()] | None = None + + +TestSegmentEqualConstraints = Annotated[ + TaggedVariantA | TaggedVariantB, + Field( + description="Union whose members share a field with equal-but-distinct constraint instances" + ), +] + + +class LiteralSubtypeModel(BaseModel): + """Model with a required Literal field and an optional string.""" + + subtype: Literal["a", "b", "c"] + name: str | None = None + + +class TripleInnerModel(BaseModel): + tag: Annotated[str, MinLen(1)] + + +class TripleNestedArrayModel(BaseModel): + deep: list[list[list[TripleInnerModel]]] + + +@radio_group("a", "b") +class RadioModel(BaseModel): + a: bool = False + b: bool = False + + +@require_any_of("x", "y") +class RequireAnyModel(BaseModel): + x: str | None = None + y: str | None = None + + +@require_any_true( + FieldEqCondition("is_land", True), + FieldEqCondition("is_territorial", True), +) +class RequireAnyTrueModel(BaseModel): + is_land: bool | None = None + is_territorial: bool | None = None + + +def discover_feature(class_name: str) -> ModelSpec: + """Discover and extract a model spec by class name.""" + models = discover_models() + for key, entry in models.items(): + if (is_model_class(entry) and entry.__name__ == class_name) or ( + is_union_alias(entry) and entry_point_class(key.entry_point) == class_name + ): + spec = extract_model_spec(key, entry) + if spec is not None: + return spec + raise LookupError(f"{class_name} not found in discovered models") + + +def spec_for_model( + cls: type[BaseModel], + *, + entry_point: str | None = None, + partitions: Mapping[str, str] | None = None, +) -> RecordSpec: + """Extract a model class for tests; sub-specs are populated by extract_model.""" + return extract_model(cls, entry_point=entry_point, partitions=partitions) + + +def union_spec_for(name: str, union_type: object) -> UnionSpec: + """Extract a discriminated-union annotation for tests.""" + return extract_union(name, union_type) def assert_golden(actual: str, golden_path: Path, *, update: bool) -> None: diff --git a/packages/overture-schema-codegen/tests/conftest.py b/packages/overture-schema-codegen/tests/conftest.py index d66cf72a3..fbd1f0f1e 100644 --- a/packages/overture-schema-codegen/tests/conftest.py +++ b/packages/overture-schema-codegen/tests/conftest.py @@ -6,7 +6,7 @@ from codegen_test_support import find_model_class from overture.schema.codegen.extraction.model_extraction import extract_model from overture.schema.codegen.extraction.numeric_extraction import extract_numerics -from overture.schema.codegen.extraction.specs import ModelSpec +from overture.schema.codegen.extraction.specs import RecordSpec from overture.schema.codegen.markdown.pipeline import ( partition_numeric_and_geometry_types, ) @@ -52,7 +52,7 @@ def building_class(all_discovered_models: dict) -> type[BaseModel]: @pytest.fixture -def building_spec(building_class: type[BaseModel]) -> ModelSpec: +def building_spec(building_class: type[BaseModel]) -> RecordSpec: """Extract the Building model spec.""" return extract_model(building_class) diff --git a/packages/overture-schema-codegen/tests/test_cli.py b/packages/overture-schema-codegen/tests/test_cli.py index 606e1837f..07120d132 100644 --- a/packages/overture-schema-codegen/tests/test_cli.py +++ b/packages/overture-schema-codegen/tests/test_cli.py @@ -7,7 +7,7 @@ import pytest from click.testing import CliRunner from overture.schema.codegen.cli import cli -from overture.schema.codegen.extraction.specs import ModelSpec +from overture.schema.codegen.extraction.specs import RecordSpec class TestCliList: @@ -376,10 +376,10 @@ class TestCliEntryPoint: def test_generate_sets_entry_point_on_specs( self, cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch ) -> None: - captured: list[ModelSpec] = [] + captured: list[RecordSpec] = [] - def spy(feature_specs: list, schema_root: str, output_dir: object) -> None: - captured.extend(feature_specs) + def spy(model_specs: list, schema_root: str, output_dir: object) -> None: + captured.extend(model_specs) monkeypatch.setattr("overture.schema.codegen.cli._generate_markdown", spy) result = cli_runner.invoke( @@ -438,6 +438,62 @@ def test_segment_appears_in_markdown_output( assert "subtype" in content +class TestCliGeneratePyspark: + def test_pyspark_format_accepted(self, cli_runner: CliRunner) -> None: + """pyspark format should be a valid --format choice.""" + result = cli_runner.invoke(cli, ["generate", "--format", "pyspark"]) + assert "Invalid value" not in (result.output or "") + assert result.exit_code == 0 + + def test_pyspark_to_output_dir(self, cli_runner: CliRunner, tmp_path: Path) -> None: + """pyspark format with --output-dir should create expression files.""" + result = cli_runner.invoke( + cli, + [ + "generate", + "--format", + "pyspark", + "--tag", + "overture:theme=divisions", + "--output-dir", + str(tmp_path), + ], + ) + assert result.exit_code == 0 + py_files = list(tmp_path.rglob("*.py")) + assert len(py_files) > 0 + names = {f.stem for f in py_files} + assert "division_area" in names + + def test_pyspark_writes_under_entry_point_namespace( + self, cli_runner: CliRunner, tmp_path: Path + ) -> None: + """Expression modules land under the entry-point namespace, no extra `expressions/` wrapper.""" + output_dir = tmp_path / "expressions" + result = cli_runner.invoke( + cli, + [ + "generate", + "--format", + "pyspark", + "--tag", + "overture:theme=divisions", + "--output-dir", + str(output_dir), + ], + ) + assert result.exit_code == 0 + + # Modules land under the entry-point namespace. + assert (output_dir / "overture" / "schema" / "divisions").is_dir() + + # No nested expressions/ subdirectory. + nested = output_dir / "expressions" + assert not nested.exists(), ( + f"Nested expressions/ directory found: {list(nested.iterdir())}" + ) + + class TestReverseReferences: """Integration test: Reverse references appear in generated markdown.""" diff --git a/packages/overture-schema-codegen/tests/test_constraint_description.py b/packages/overture-schema-codegen/tests/test_constraint_description.py index b205f6f73..941537a2e 100644 --- a/packages/overture-schema-codegen/tests/test_constraint_description.py +++ b/packages/overture-schema-codegen/tests/test_constraint_description.py @@ -1,15 +1,30 @@ """Tests for constraint description (model-level and field-level).""" -from annotated_types import Ge, Gt, Interval, Le, Lt, MaxLen, MinLen +import re + +from annotated_types import Ge, Gt, Interval, Le, Lt from overture.schema.codegen.extraction.field_constraints import ( constraint_display_text, describe_field_constraint, ) +from overture.schema.codegen.extraction.length_constraints import ( + ArrayMaxLen, + ArrayMinLen, + ScalarMaxLen, + ScalarMinLen, +) +from overture.schema.codegen.extraction.literal_alternatives import ( + LiteralAlternatives, +) from overture.schema.codegen.extraction.model_constraints import ( analyze_model_constraints, ) from overture.schema.codegen.extraction.specs import TypeIdentity from overture.schema.codegen.extraction.type_analyzer import ConstraintSource +from overture.schema.system.field_constraint.string import ( + CountryCodeAlpha2Constraint, + PatternConstraint, +) from overture.schema.system.model_constraint import ( FieldEqCondition, ForbidIfConstraint, @@ -153,6 +168,24 @@ class FutureConstraint(ModelConstraint): assert result == ["`@future_thing`"] +class TestLiteralAlternativesProse: + """`X | Literal[c]` renders a faithful 'Also accepts' note in the docs.""" + + def test_empty_string_literal(self) -> None: + assert ( + describe_field_constraint(LiteralAlternatives(("",))) + == "Also accepts: `''`" + ) + + def test_named_literal(self) -> None: + result = describe_field_constraint(LiteralAlternatives(("Global",))) + assert result == "Also accepts: `'Global'`" + + def test_multiple_literals(self) -> None: + result = describe_field_constraint(LiteralAlternatives(("a", "b"))) + assert result == "Also accepts: `'a'`, `'b'`" + + class TestConsolidation: """Consolidation of same-field conditional constraints.""" @@ -377,11 +410,27 @@ def test_gt(self) -> None: def test_lt(self) -> None: assert describe_field_constraint(Lt(lt=100)) == "`< 100`" - def test_min_len(self) -> None: - assert describe_field_constraint(MinLen(min_length=1)) == "Minimum length: 1" + def test_scalar_min_len(self) -> None: + assert ( + describe_field_constraint(ScalarMinLen(min_length=1)) == "Minimum length: 1" + ) - def test_max_len(self) -> None: - assert describe_field_constraint(MaxLen(max_length=10)) == "Maximum length: 10" + def test_array_min_len(self) -> None: + assert ( + describe_field_constraint(ArrayMinLen(min_length=1)) == "Minimum length: 1" + ) + + def test_scalar_max_len(self) -> None: + assert ( + describe_field_constraint(ScalarMaxLen(max_length=10)) + == "Maximum length: 10" + ) + + def test_array_max_len(self) -> None: + assert ( + describe_field_constraint(ArrayMaxLen(max_length=10)) + == "Maximum length: 10" + ) def test_interval_closed(self) -> None: assert describe_field_constraint(Interval(ge=0, le=100)) == "`0 ≤ x ≤ 100`" @@ -494,3 +543,29 @@ def link_fn(tid: TypeIdentity) -> str: assert len(received) == 1 assert received[0].obj is Target assert result == "References [`Target`](link) (composition)" + + +class TestConstraintPatternFlags: + """constraint_display_text surfaces a compiled pattern's regex flags.""" + + def _display(self, constraint: object) -> str: + cs = ConstraintSource(source_ref=None, source_name=None, constraint=constraint) + return constraint_display_text(cs) + + def test_case_insensitive_pattern_shows_inline_flag(self) -> None: + # A case-insensitive pattern displayed without its flag misleads the + # reader into thinking only lowercase matches. + c = PatternConstraint(r"^[a-z]+$", "err: {value}", flags=re.I) + assert "pattern: `(?i)^[a-z]+$`" in self._display(c) + + def test_unflagged_pattern_omits_inline_flag_group(self) -> None: + # re.UNICODE is the implicit str-pattern default and must not leak as + # a (?u) group onto every pattern. + assert "pattern: `^[A-Z]{2}$`" in self._display(CountryCodeAlpha2Constraint()) + assert "(?" not in self._display(CountryCodeAlpha2Constraint()) + + def test_multiple_flags_render_as_one_group(self) -> None: + # Display tolerates flags pyspark cannot honor (re.M); doc generation + # must not crash where check generation would. + c = PatternConstraint(r"^[a-z]+$", "err: {value}", flags=re.I | re.M) + assert "pattern: `(?im)^[a-z]+$`" in self._display(c) diff --git a/packages/overture-schema-codegen/tests/test_example_loader.py b/packages/overture-schema-codegen/tests/test_example_loader.py index 1f94bc06d..541b81282 100644 --- a/packages/overture-schema-codegen/tests/test_example_loader.py +++ b/packages/overture-schema-codegen/tests/test_example_loader.py @@ -9,6 +9,7 @@ from typing import Annotated, Literal import pytest +from overture.schema.buildings.building import Building from overture.schema.codegen.extraction.examples import ( ExampleRecord, _inject_literal_fields, @@ -20,7 +21,14 @@ resolve_pyproject_path, validate_example, ) +from overture.schema.system.primitive import BBox, Geometry +from overture.schema.transportation import Segment +from overture.schema.transportation.segment.models import ( + RoadSegment, + TransportationSegment, +) from pydantic import BaseModel, ConfigDict, Field, Tag, ValidationError +from shapely.geometry import Point class TestOrderExampleRows: @@ -639,10 +647,6 @@ class TestIntegration: def test_real_building_examples_validate(self) -> None: """Validate real Building examples from the schema package.""" - pytest.importorskip("overture.schema.buildings.building") - - from overture.schema.buildings.building import Building # noqa: PLC0415 - pyproject_path = resolve_pyproject_path(Building) assert pyproject_path is not None, "Could not find pyproject.toml for Building" @@ -657,14 +661,6 @@ def test_real_building_examples_validate(self) -> None: def test_real_segment_examples_validate(self) -> None: """Validate real Segment examples (discriminated union with cross-arm fields).""" - pytest.importorskip("overture.schema.transportation") - - from overture.schema.transportation import Segment # noqa: PLC0415 - from overture.schema.transportation.segment.models import ( # noqa: PLC0415 - RoadSegment, - TransportationSegment, - ) - pyproject_path = resolve_pyproject_path(RoadSegment) assert pyproject_path is not None @@ -889,7 +885,6 @@ class Aliased(BaseModel): def test_slots_based_field_flattened(self) -> None: """Non-BaseModel types with __slots__ and properties are flattened.""" - from overture.schema.system.primitive import BBox # noqa: PLC0415 class WithBBox(BaseModel): id: str @@ -907,7 +902,6 @@ class WithBBox(BaseModel): def test_none_slots_based_field_is_leaf(self) -> None: """A slots-based field with None value is a leaf.""" - from overture.schema.system.primitive import BBox # noqa: PLC0415 class WithBBox(BaseModel): id: str @@ -919,8 +913,6 @@ class WithBBox(BaseModel): def test_single_slot_wrapper_is_leaf(self) -> None: """Single-slot types (wrappers like Geometry) are leaf values.""" - from overture.schema.system.primitive import Geometry # noqa: PLC0415 - from shapely.geometry import Point # noqa: PLC0415 class WithGeom(BaseModel): id: str diff --git a/packages/overture-schema-codegen/tests/test_field_walk.py b/packages/overture-schema-codegen/tests/test_field_walk.py new file mode 100644 index 000000000..3cdb08048 --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_field_walk.py @@ -0,0 +1,235 @@ +"""Tests for the `FieldShape` walker and structural helpers.""" + +import enum + +import pytest +from overture.schema.codegen.extraction.field import ( + AnyScalar, + ArrayOf, + ConstraintSource, + LiteralScalar, + MapOf, + ModelRef, + NewTypeShape, + Primitive, + UnionRef, +) +from overture.schema.codegen.extraction.field_walk import ( + enum_source, + map_key_value_constraints, + shape_children, + terminal_model_ref, + terminal_of, + terminal_primitive, + terminal_scalar, + walk_shape, +) + + +class TestShapeChildren: + """Direct child enumeration over `FieldShape`.""" + + def test_scalar_has_no_children(self) -> None: + assert list(shape_children(Primitive(base_type="str"))) == [] + + def test_array_yields_element(self) -> None: + inner = Primitive(base_type="int32") + assert list(shape_children(ArrayOf(element=inner))) == [inner] + + def test_map_yields_key_then_value(self) -> None: + k = Primitive(base_type="str") + v = Primitive(base_type="int32") + assert list(shape_children(MapOf(key=k, value=v))) == [k, v] + + def test_model_ref_has_no_children(self) -> None: + sentinel = object() + assert list(shape_children(ModelRef(model=sentinel))) == [] # type: ignore[arg-type] + + def test_union_ref_has_no_children(self) -> None: + sentinel = object() + assert list(shape_children(UnionRef(union=sentinel))) == [] # type: ignore[arg-type] + + def test_newtype_shape_yields_inner(self) -> None: + inner = Primitive(base_type="int32") + nt = NewTypeShape(name="N", ref=object(), inner=inner) + assert list(shape_children(nt)) == [inner] + + +class TestWalkShape: + """Pre-order traversal over `FieldShape` trees.""" + + @staticmethod + def _collect(root: object) -> list[object]: + seen: list[object] = [] + walk_shape(root, seen.append) # type: ignore[arg-type] + return seen + + def test_scalar_visits_once(self) -> None: + root = Primitive(base_type="str") + assert self._collect(root) == [root] + + def test_nested_arrays(self) -> None: + leaf = Primitive(base_type="int32") + middle = ArrayOf(element=leaf) + root = ArrayOf(element=middle) + assert self._collect(root) == [root, middle, leaf] + + def test_map_visits_self_key_value(self) -> None: + k = Primitive(base_type="str") + v = Primitive(base_type="int32") + root = MapOf(key=k, value=v) + assert self._collect(root) == [root, k, v] + + def test_model_ref_is_boundary(self) -> None: + sentinel = object() + root = ModelRef(model=sentinel) # type: ignore[arg-type] + assert self._collect(root) == [root] + + def test_union_ref_is_boundary(self) -> None: + sentinel = object() + root = UnionRef(union=sentinel) # type: ignore[arg-type] + assert self._collect(root) == [root] + + def test_array_of_newtype_walks_through(self) -> None: + leaf = Primitive(base_type="str") + nt = NewTypeShape(name="N", ref=object(), inner=leaf) + root = ArrayOf(element=nt) + assert self._collect(root) == [root, nt, leaf] + + +_STR = Primitive(base_type="str") +_INT = Primitive(base_type="int32") +_LITERAL = LiteralScalar(values=("a",)) +_ANY = AnyScalar() +_MODEL = ModelRef(model=object()) # type: ignore[arg-type] +_MAP = MapOf(key=_STR, value=_INT) +_NEWTYPE_STR = NewTypeShape(name="N", ref=object(), inner=_STR) +_ARRAY_NEWTYPE_STR = ArrayOf(element=_NEWTYPE_STR) + + +class TestTerminalFilters: + """`terminal_of` and the three typed `terminal_*` narrowing helpers.""" + + @pytest.mark.parametrize( + ("shape", "expected"), + [ + (_STR, _STR), + (ArrayOf(element=ArrayOf(element=_INT)), _INT), + (_NEWTYPE_STR, _STR), + (_ARRAY_NEWTYPE_STR, _STR), + (ArrayOf(element=_MODEL), _MODEL), + (_MAP, _MAP), + ], + ) + def test_terminal_of_unwraps_to_innermost( + self, shape: object, expected: object + ) -> None: + assert terminal_of(shape) is expected # type: ignore[arg-type] + + @pytest.mark.parametrize( + ("shape", "expected"), + [ + (_STR, _STR), + (ArrayOf(element=_INT), _INT), + (_NEWTYPE_STR, _STR), + (_LITERAL, None), + (_ANY, None), + (_MODEL, None), + ], + ) + def test_terminal_primitive(self, shape: object, expected: object) -> None: + assert terminal_primitive(shape) is expected # type: ignore[arg-type] + + @pytest.mark.parametrize( + ("shape", "expected"), + [ + (_STR, _STR), + (_LITERAL, _LITERAL), + (_ANY, _ANY), + (ArrayOf(element=_LITERAL), _LITERAL), + (_MODEL, None), + (_MAP, None), + ], + ) + def test_terminal_scalar(self, shape: object, expected: object) -> None: + assert terminal_scalar(shape) is expected # type: ignore[arg-type] + + @pytest.mark.parametrize( + ("shape", "expected"), + [ + (_MODEL, _MODEL), + (ArrayOf(element=_MODEL), _MODEL), + (NewTypeShape(name="N", ref=object(), inner=_MODEL), _MODEL), + (_STR, None), + (_LITERAL, None), + (_ANY, None), + ], + ) + def test_terminal_model_ref(self, shape: object, expected: object) -> None: + assert terminal_model_ref(shape) is expected # type: ignore[arg-type] + + +class _Color(enum.Enum): + RED = "red" + BLUE = "blue" + + +class TestEnumSource: + """`enum_source` extracts the `Enum` class from an enum-backed `Primitive`.""" + + def test_enum_backed_primitive_returns_class(self) -> None: + shape = Primitive(base_type="str", source_type=_Color) + assert enum_source(shape) is _Color + + def test_plain_primitive_returns_none(self) -> None: + shape = Primitive(base_type="str") + assert enum_source(shape) is None + + def test_literal_scalar_returns_none(self) -> None: + shape = LiteralScalar(values=("a",)) + assert enum_source(shape) is None + + def test_non_enum_class_source_type_returns_none(self) -> None: + # source_type is a real class that is not an Enum subclass + shape = Primitive(base_type="str", source_type=int) + assert enum_source(shape) is None + + def test_newtype_wrapping_enum_primitive_returns_none(self) -> None: + # wrappers are not unwrapped — only a bare Primitive matches + inner = Primitive(base_type="str", source_type=_Color) + nt = NewTypeShape(name="ColorAlias", ref=object(), inner=inner) + assert enum_source(nt) is None + + def test_array_of_enum_primitive_returns_none(self) -> None: + inner = Primitive(base_type="str", source_type=_Color) + shape = ArrayOf(element=inner) + assert enum_source(shape) is None + + +def _constraint() -> ConstraintSource: + """A directly-applied constraint with no NewType source.""" + return ConstraintSource(source_ref=None, source_name=None, constraint=object()) + + +class TestMapKeyValueConstraints: + """`map_key_value_constraints` collects a `MapOf` terminal's sides.""" + + def test_direct_map_returns_key_and_value_constraints(self) -> None: + kc, vc = _constraint(), _constraint() + shape = MapOf( + key=Primitive(base_type="str", constraints=(kc,)), + value=Primitive(base_type="int32", constraints=(vc,)), + ) + assert map_key_value_constraints(shape) == ((kc,), (vc,)) + + def test_looks_through_newtype_and_array_wrappers(self) -> None: + vc = _constraint() + inner_map = MapOf( + key=Primitive(base_type="str"), + value=Primitive(base_type="int32", constraints=(vc,)), + ) + shape = NewTypeShape(name="N", ref=object(), inner=ArrayOf(element=inner_map)) + assert map_key_value_constraints(shape) == ((), (vc,)) + + def test_non_map_shape_returns_empty(self) -> None: + assert map_key_value_constraints(Primitive(base_type="str")) == ((), ()) diff --git a/packages/overture-schema-codegen/tests/test_golden_markdown.py b/packages/overture-schema-codegen/tests/test_golden_markdown.py index 42320ee69..cffdc2def 100644 --- a/packages/overture-schema-codegen/tests/test_golden_markdown.py +++ b/packages/overture-schema-codegen/tests/test_golden_markdown.py @@ -18,20 +18,17 @@ Venue, Widget, assert_golden, + spec_for_model, ) from overture.schema.codegen.extraction.enum_extraction import extract_enum -from overture.schema.codegen.extraction.model_extraction import ( - expand_model_tree, - extract_model, -) from overture.schema.codegen.extraction.newtype_extraction import extract_newtype -from overture.schema.codegen.extraction.specs import TypeIdentity +from overture.schema.codegen.extraction.specs import ModelSpec, TypeIdentity from overture.schema.codegen.layout.type_collection import ( collect_all_supplementary_types, ) from overture.schema.codegen.markdown.renderer import ( render_enum, - render_feature, + render_model, render_newtype, ) from overture.schema.codegen.markdown.reverse_references import ( @@ -67,15 +64,13 @@ @pytest.fixture(scope="module") def reverse_refs() -> dict[TypeIdentity, list[UsedByEntry]]: """Compute reverse references for all test models.""" - feature_specs = [] + model_specs: list[ModelSpec] = [] for model_class, _ in FEATURE_CASES: assert isinstance(model_class, type) and issubclass(model_class, BaseModel) - spec = extract_model(model_class) - expand_model_tree(spec) - feature_specs.append(spec) + model_specs.append(spec_for_model(model_class)) - all_specs = collect_all_supplementary_types(feature_specs) - return compute_reverse_references(feature_specs, all_specs) + all_specs = collect_all_supplementary_types(model_specs) + return compute_reverse_references(model_specs, all_specs) @pytest.mark.parametrize( @@ -89,10 +84,9 @@ def test_feature_golden( update_golden: bool, reverse_refs: dict[TypeIdentity, list[UsedByEntry]], ) -> None: - spec = extract_model(model_class) - expand_model_tree(spec) + spec = spec_for_model(model_class) used_by = reverse_refs.get(spec.identity) - actual = render_feature(spec, used_by=used_by) + actual = render_model(spec, used_by=used_by) assert_golden(actual, GOLDEN_DIR / golden_filename, update=update_golden) diff --git a/packages/overture-schema-codegen/tests/test_integration_real_models.py b/packages/overture-schema-codegen/tests/test_integration_real_models.py index 3a8214359..85ac6f718 100644 --- a/packages/overture-schema-codegen/tests/test_integration_real_models.py +++ b/packages/overture-schema-codegen/tests/test_integration_real_models.py @@ -5,21 +5,18 @@ """ import pytest -from codegen_test_support import assert_literal_field +from codegen_test_support import assert_literal_field, spec_for_model from overture.schema.codegen.extraction.model_extraction import extract_model from overture.schema.codegen.extraction.specs import ( - FeatureSpec, ModelSpec, + RecordSpec, UnionSpec, filter_model_classes, - is_model_class, - is_union_alias, ) -from overture.schema.codegen.extraction.type_analyzer import TypeKind from overture.schema.codegen.extraction.union_extraction import extract_union -from overture.schema.codegen.layout.module_layout import entry_point_class from overture.schema.codegen.markdown.pipeline import generate_markdown_pages -from overture.schema.codegen.markdown.renderer import render_feature +from overture.schema.codegen.markdown.renderer import render_model +from overture.schema.codegen.spec_discovery import extract_model_spec from overture.schema.system.discovery import discover_models from overture.schema.transportation import Segment from overture.schema.transportation.segment.models import RoadSegment @@ -29,21 +26,6 @@ class TestDiscoverModels: """Tests for model discovery.""" - def test_discover_models_returns_dict(self) -> None: - """discover_models() should return a dictionary.""" - models = discover_models() - assert isinstance(models, dict) - - def test_discover_models_finds_building( - self, building_class: type[BaseModel] - ) -> None: - """Should discover the Building model.""" - assert issubclass(building_class, BaseModel) - - def test_discover_models_finds_place(self, place_class: type[BaseModel]) -> None: - """Should discover the Place model.""" - assert issubclass(place_class, BaseModel) - def test_discover_models_returns_multiple_themes(self) -> None: """Should discover models from multiple themes.""" models = discover_models() @@ -53,26 +35,25 @@ def test_discover_models_returns_multiple_themes(self) -> None: class TestExtractBuildingModel: """Tests for extracting the Building model.""" - def test_extract_building_has_name(self, building_spec: ModelSpec) -> None: + def test_extract_building_has_name(self, building_spec: RecordSpec) -> None: """Building model spec should have correct name.""" assert building_spec.name == "Building" - def test_extract_building_has_theme_type(self, building_spec: ModelSpec) -> None: + def test_extract_building_has_theme_type(self, building_spec: RecordSpec) -> None: """Building should have theme='buildings', type='building' as Literal fields.""" assert_literal_field(building_spec, "theme", "buildings") assert_literal_field(building_spec, "type", "building") - def test_extract_building_has_fields(self, building_spec: ModelSpec) -> None: + def test_extract_building_has_fields(self, building_spec: RecordSpec) -> None: """Building should have multiple fields.""" assert len(building_spec.fields) > 0, "Building should have at least one field" field_names = {f.name for f in building_spec.fields} assert "id" in field_names - def test_building_field_types_are_valid(self, building_spec: ModelSpec) -> None: - """All Building fields should have valid TypeInfo.""" + def test_building_field_shapes_are_present(self, building_spec: RecordSpec) -> None: + """Every Building field has a `FieldShape`.""" for field in building_spec.fields: - assert field.type_info is not None - assert field.type_info.kind in TypeKind + assert field.shape is not None class TestExtractPlaceModel: @@ -109,17 +90,12 @@ def test_no_analyze_type_crashes(self, all_discovered_models: dict) -> None: spec = extract_model(model_class) assert spec.name == model_class.__name__ - def test_all_field_types_resolved(self, all_discovered_models: dict) -> None: - """All fields should have resolved TypeInfo.""" + def test_all_field_shapes_resolved(self, all_discovered_models: dict) -> None: + """Every field of every discovered model carries a `FieldShape`.""" for model_class in filter_model_classes(all_discovered_models): spec = extract_model(model_class) for field in spec.fields: - assert field.type_info.base_type, ( - f"No base_type for {spec.name}.{field.name}" - ) - assert field.type_info.kind in TypeKind, ( - f"Invalid kind for {spec.name}.{field.name}" - ) + assert field.shape is not None, f"No shape for {spec.name}.{field.name}" class TestMarkdownRenderingRealModels: @@ -127,7 +103,7 @@ class TestMarkdownRenderingRealModels: def test_render_building_content(self, building_class: type[BaseModel]) -> None: """Building renders with title, field table, and expected fields.""" - markdown = render_feature(extract_model(building_class)) + markdown = render_model(spec_for_model(building_class)) assert "# Building" in markdown assert "| Name |" in markdown @@ -136,11 +112,9 @@ def test_render_building_content(self, building_class: type[BaseModel]) -> None: assert "geometry" in markdown def test_render_all_models_without_crash(self, all_discovered_models: dict) -> None: - """render_feature should not crash on any discovered model.""" + """render_model should not crash on any discovered model.""" for model_class in filter_model_classes(all_discovered_models): - markdown = render_feature(extract_model(model_class)) - assert isinstance(markdown, str) - assert len(markdown) > 0 + render_model(spec_for_model(model_class)) class TestDiscriminatedUnions: @@ -221,9 +195,8 @@ def test_segment_discriminator_extracted_from_callable( assert segment_spec.discriminator_field == "subtype" assert segment_spec.discriminator_mapping is not None assert len(segment_spec.discriminator_mapping) == 3 - # Keys are str(enum_member), e.g. "Subtype.ROAD" - road_key = next(k for k in segment_spec.discriminator_mapping if "ROAD" in k) - assert segment_spec.discriminator_mapping[road_key] is RoadSegment + # Keys are runtime string values, e.g. "road" + assert segment_spec.discriminator_mapping["road"] is RoadSegment def test_segment_common_base_is_base_model(self, segment_spec: UnionSpec) -> None: """Segment common_base is the shared base class.""" @@ -234,28 +207,22 @@ def test_segment_common_base_is_base_model(self, segment_spec: UnionSpec) -> Non assert "id" in segment_spec.common_base.model_fields -@pytest.fixture(scope="module") -def pages() -> list: - """Generate all pages from real discovered models.""" - models = discover_models() - feature_specs: list[FeatureSpec] = [] - for key, entry in models.items(): - if is_model_class(entry): - feature_specs.append(extract_model(entry, entry_point=key.entry_point)) - elif is_union_alias(entry): - feature_specs.append( - extract_union( - entry_point_class(key.entry_point), - entry, - entry_point=key.entry_point, - ) - ) - return generate_markdown_pages(feature_specs, "overture.schema") - - class TestPydanticTypePages: """End-to-end: pipeline produces pages for referenced Pydantic built-in types.""" + _SCHEMA_ROOT = "overture.schema" + + @pytest.fixture(scope="class") + def pages(self) -> list: + """Generate all pages from real discovered models.""" + models = discover_models() + model_specs: list[ModelSpec] = [ + spec + for key, entry in models.items() + if (spec := extract_model_spec(key, entry)) is not None + ] + return generate_markdown_pages(model_specs, self._SCHEMA_ROOT) + def test_http_url_page_exists(self, pages: list) -> None: """Pipeline produces a page for HttpUrl under pydantic/networks/.""" paths = {str(p.path) for p in pages} @@ -274,5 +241,5 @@ def test_http_url_page_content(self, pages: list) -> None: def test_place_links_to_http_url(self, pages: list) -> None: """Place feature page links to the HttpUrl type page.""" - place_page = next(p for p in pages if p.path.stem == "place" and p.is_feature) + place_page = next(p for p in pages if p.path.stem == "place" and p.is_model) assert "HttpUrl" in place_page.content diff --git a/packages/overture-schema-codegen/tests/test_markdown_renderer.py b/packages/overture-schema-codegen/tests/test_markdown_renderer.py index 698f9d70a..271347ca2 100644 --- a/packages/overture-schema-codegen/tests/test_markdown_renderer.py +++ b/packages/overture-schema-codegen/tests/test_markdown_renderer.py @@ -22,12 +22,10 @@ TreeNode, Venue, make_union_spec, + spec_for_model, ) from overture.schema.codegen.extraction.examples import ExampleRecord -from overture.schema.codegen.extraction.model_extraction import ( - expand_model_tree, - extract_model, -) +from overture.schema.codegen.extraction.model_extraction import extract_model from overture.schema.codegen.extraction.newtype_extraction import extract_newtype from overture.schema.codegen.extraction.specs import ( AnnotatedField, @@ -45,7 +43,7 @@ _linkify_bare_urls, _sanitize_for_table_cell, render_enum, - render_feature, + render_model, render_newtype, render_primitives_from_specs, render_pydantic_type, @@ -186,12 +184,12 @@ def test_mixed_code_span_and_bare_url(self) -> None: class TestRenderFeatureBasic: - """Tests for render_feature with basic models.""" + """Tests for render_model with basic models.""" def test_renders_title_from_model_name(self) -> None: """Should render model name as H1 title.""" spec = extract_model(SimpleModel) - result = render_feature(spec) + result = render_model(spec) assert "# SimpleModel" in result @@ -204,7 +202,7 @@ class DescribedModel(BaseModel): value: int spec = extract_model(DescribedModel) - result = render_feature(spec) + result = render_model(spec) assert "This is the model description." in result @@ -217,7 +215,7 @@ class ModelWithField(BaseModel): name: str spec = extract_model(ModelWithField) - result = render_feature(spec) + result = render_model(spec) assert "## Fields" in result @@ -230,7 +228,7 @@ class ModelWithField(BaseModel): name: str spec = extract_model(ModelWithField) - result = render_feature(spec) + result = render_model(spec) assert "| Name | Type | Description |" in result assert "| -----: | :----: | ------------- |" in result @@ -248,7 +246,7 @@ class ModelWithRequired(BaseModel): name: str = Field(description="The name") spec = extract_model(ModelWithRequired) - result = render_feature(spec) + result = render_model(spec) assert "| `name` |" in result assert "| `string` |" in result @@ -263,7 +261,7 @@ class ModelWithOptional(BaseModel): nickname: str | None = Field(None, description="Optional nickname") spec = extract_model(ModelWithOptional) - result = render_feature(spec) + result = render_model(spec) assert "| `nickname` |" in result assert "(optional)" in result @@ -280,7 +278,7 @@ class ModelWithTypes(BaseModel): active: bool spec = extract_model(ModelWithTypes) - result = render_feature(spec) + result = render_model(spec) # Check that fields are present (exact type format may vary) assert "`count`" in result @@ -296,7 +294,7 @@ class ModelWithMultilineDesc(BaseModel): name: str = Field(description="First line.\n\nSecond paragraph.") spec = extract_model(ModelWithMultilineDesc) - result = render_feature(spec) + result = render_model(spec) assert "First line.

Second paragraph." in result # The table should not be broken by a blank line @@ -320,7 +318,7 @@ class Place(FeatureBase[Literal["places"], Literal["place"]]): name: str spec = extract_model(Place) - result = render_feature(spec) + result = render_model(spec) # Theme and type should appear somewhere in output assert "places" in result @@ -339,7 +337,7 @@ class TestFeature(FeatureBase[Literal["test_theme"], Literal["test_type"]]): name: str spec = extract_model(TestFeature) - result = render_feature(spec) + result = render_model(spec) assert '| `"test_theme"` |' in result assert '| `"test_type"` |' in result @@ -365,9 +363,7 @@ class ModelWithSources(BaseModel): sources: TestSources | None = None - spec = extract_model(ModelWithSources) - expand_model_tree(spec) - result = render_feature(spec) + result = render_model(spec_for_model(ModelWithSources)) assert "`TestSources`" in result assert "(list, optional)" in result @@ -381,7 +377,7 @@ class ModelWithColor(BaseModel): color: HexColor | None = None spec = extract_model(ModelWithColor) - result = render_feature(spec) + result = render_model(spec) assert "`HexColor`" in result assert "(optional)" in result @@ -395,7 +391,7 @@ class ModelWithCount(BaseModel): count: int32 spec = extract_model(ModelWithCount) - result = render_feature(spec) + result = render_model(spec) assert "| `int32` |" in result # Should NOT be linked @@ -410,7 +406,7 @@ class ModelWithName(BaseModel): name: str spec = extract_model(ModelWithName) - result = render_feature(spec) + result = render_model(spec) assert "| `string` |" in result @@ -426,7 +422,7 @@ class ModelWithEnum(BaseModel): status: Status spec = extract_model(ModelWithEnum) - result = render_feature(spec) + result = render_model(spec) assert "| `Status` |" in result @@ -441,9 +437,7 @@ class Outer(BaseModel): inner: Inner - spec = extract_model(Outer) - expand_model_tree(spec) - result = render_feature(spec) + result = render_model(spec_for_model(Outer)) assert "| `Inner` |" in result @@ -453,9 +447,7 @@ class TestRenderFeatureInlineExpansion: def test_direct_model_fields_expanded_with_dot_prefix(self) -> None: """Direct model field expands sub-fields with dot notation.""" - spec = extract_model(FeatureWithAddress) - expand_model_tree(spec) - result = render_feature(spec) + result = render_model(spec_for_model(FeatureWithAddress)) assert "| `address.street` |" in result assert "| `address.city` |" in result @@ -463,18 +455,14 @@ def test_direct_model_fields_expanded_with_dot_prefix(self) -> None: def test_list_of_model_fields_expanded_with_bracket_dot_prefix(self) -> None: """List-of-model field expands sub-fields with []. notation.""" - spec = extract_model(FeatureWithSources) - expand_model_tree(spec) - result = render_feature(spec) + result = render_model(spec_for_model(FeatureWithSources)) assert "| `sources[]` |" in result assert "| `sources[].dataset` |" in result def test_cycle_detection_prevents_infinite_recursion(self) -> None: """Recursive model emits parent row but does not recurse.""" - spec = extract_model(TreeNode) - expand_model_tree(spec) - result = render_feature(spec) + result = render_model(spec_for_model(TreeNode)) # The parent field row appears assert "| `parent` |" in result @@ -484,16 +472,14 @@ def test_cycle_detection_prevents_infinite_recursion(self) -> None: def test_primitive_field_unchanged(self) -> None: """Primitive fields produce a single row without expansion.""" spec = extract_model(SimpleModel) - result = render_feature(spec) + result = render_model(spec) lines = [line for line in result.splitlines() if "| `name` |" in line] assert len(lines) == 1 def test_parent_row_preserved_before_expansion(self) -> None: """The parent field row still appears before expanded sub-fields.""" - spec = extract_model(FeatureWithAddress) - expand_model_tree(spec) - result = render_feature(spec) + result = render_model(spec_for_model(FeatureWithAddress)) # Parent row for 'address' itself appears assert "| `address` |" in result @@ -514,7 +500,7 @@ class TestRenderFeatureConstraints: def test_venue_has_constraints_section(self) -> None: """Venue's @require_any_of renders as a Constraints section.""" spec = extract_model(Venue) - result = render_feature(spec) + result = render_model(spec) assert "## Constraints" in result assert "At least one of `name`, `description` must be set" in result @@ -523,7 +509,7 @@ def test_constraints_section_between_fields_and_examples(self) -> None: """Constraints section appears after Fields, before Examples.""" spec = extract_model(Venue) examples = [ExampleRecord(rows=[("name", "test")])] - result = render_feature(spec, examples=examples) + result = render_model(spec, examples=examples) lines = result.splitlines() fields_line = next(i for i, line in enumerate(lines) if "## Fields" in line) @@ -543,7 +529,7 @@ class Plain(BaseModel): name: str spec = extract_model(Plain) - result = render_feature(spec) + result = render_model(spec) assert "## Constraints" not in result @@ -557,7 +543,7 @@ class Strict(BaseModel): name: str spec = extract_model(Strict) - result = render_feature(spec) + result = render_model(spec) assert "## Constraints" not in result @@ -568,7 +554,7 @@ class TestRenderFeatureConstraintNotes: def test_venue_name_field_includes_constraint_note(self) -> None: """Venue's name field description cell includes constraint note in italics.""" spec = extract_model(Venue) - result = render_feature(spec) + result = render_model(spec) # Find the row for 'name' field lines = result.splitlines() @@ -580,7 +566,7 @@ def test_venue_name_field_includes_constraint_note(self) -> None: def test_field_with_no_description_gets_constraint_note(self) -> None: """Field with no existing description still gets the constraint note.""" spec = extract_model(Venue) - result = render_feature(spec) + result = render_model(spec) # description field on Venue has no Field(description=...) lines = result.splitlines() @@ -589,13 +575,11 @@ def test_field_with_no_description_gets_constraint_note(self) -> None: class TestRenderFeatureFieldConstraints: - """Tests for field-level constraint annotation from TypeInfo.""" + """Tests for field-level constraint annotation from the field's shape.""" def test_venue_geometry_shows_allowed_types(self) -> None: """Venue's geometry field shows GeometryTypeConstraint as a note.""" - spec = extract_model(Venue) - expand_model_tree(spec) - result = render_feature(spec) + result = render_model(spec_for_model(Venue)) lines = result.splitlines() geo_line = next(line for line in lines if "| `geometry` |" in line) @@ -603,8 +587,6 @@ def test_venue_geometry_shows_allowed_types(self) -> None: def test_venue_reference_links_when_context_available(self) -> None: """Reference constraint links the target type when LinkContext has the page.""" - spec = extract_model(Venue) - expand_model_tree(spec) ctx = LinkContext( page_path=PurePosixPath("music/venue.md"), registry={ @@ -613,7 +595,7 @@ def test_venue_reference_links_when_context_available(self) -> None: ) }, ) - result = render_feature(spec, link_ctx=ctx) + result = render_model(spec_for_model(Venue), link_ctx=ctx) lines = result.splitlines() ref_line = next(line for line in lines if "| `resident_ensemble` |" in line) @@ -622,9 +604,7 @@ def test_venue_reference_links_when_context_available(self) -> None: def test_venue_reference_unlinked_without_context(self) -> None: """Reference constraint renders as plain code when no LinkContext.""" - spec = extract_model(Venue) - expand_model_tree(spec) - result = render_feature(spec) + result = render_model(spec_for_model(Venue)) lines = result.splitlines() ref_line = next(line for line in lines if "| `resident_ensemble` |" in line) @@ -632,6 +612,66 @@ def test_venue_reference_unlinked_without_context(self) -> None: assert "aggregation, part of" in ref_line +class TestRenderFeatureMapConstraints: + """Tests for map key/value constraint notes in field description cells. + + Mirrors `Sources.license_priority`, whose value bound was previously + dropped because `all_constraints` stopped at `MapOf`. + """ + + def test_map_value_bound_shows_value_note(self) -> None: + """A directly-applied map value bound renders as a 'value:' note.""" + + class ModelWithMapValueBound(BaseModel): + """Model with a bounded map value.""" + + priorities: dict[str, Annotated[int, Field(ge=0)]] = Field( + description="Priorities." + ) + + result = render_model(extract_model(ModelWithMapValueBound)) + line = next(li for li in result.splitlines() if "| `priorities` |" in li) + assert "*value: `≥ 0`*" in line + + def test_map_key_bound_shows_key_note(self) -> None: + """A directly-applied map key bound renders as a 'key:' note.""" + + class ModelWithMapKeyBound(BaseModel): + """Model with a bounded map key.""" + + counts: dict[Annotated[int, Field(ge=1)], str] = Field( + description="Counts." + ) + + result = render_model(extract_model(ModelWithMapKeyBound)) + line = next(li for li in result.splitlines() if "| `counts` |" in li) + assert "*key: `≥ 1`*" in line + + +class TestRenderFeatureMapModelValue: + """A model-valued map names its value, never a bare `?`. + + `extract_model` resolves the value to a `ModelRef`, the case that + `resolve_type_name` could not name and silently rendered as `?`. + """ + + def test_model_valued_map_names_the_model(self) -> None: + class Inner(BaseModel): + """An inner model.""" + + x: int = Field(description="x") + + class Outer(BaseModel): + """Outer with a model-valued map.""" + + by_key: dict[str, Inner] = Field(description="Map to inner models.") + + result = render_model(extract_model(Outer)) + line = next(li for li in result.splitlines() if "| `by_key` |" in li) + assert "map" in line + assert "?" not in line + + class TestRenderEnumBasic: """Tests for render_enum with simple enums.""" @@ -827,7 +867,7 @@ class ModelWithColor(BaseModel): }, ) - result = render_feature(spec, link_ctx=ctx) + result = render_model(spec, link_ctx=ctx) assert "[`HexColor`](../../types/strings/hex_color.md)" in result @@ -853,7 +893,7 @@ class ModelWithRoof(BaseModel): }, ) - result = render_feature(spec, link_ctx=ctx) + result = render_model(spec, link_ctx=ctx) assert "[`RoofShape`](../roof_shape.md)" in result @@ -879,7 +919,7 @@ class ModelWithClass(BaseModel): }, ) - result = render_feature(spec, link_ctx=ctx) + result = render_model(spec, link_ctx=ctx) assert "[`BuildingClass`](building_class.md)" in result @@ -892,7 +932,7 @@ class ModelWithColor(BaseModel): color: HexColor | None = None spec = extract_model(ModelWithColor) - result = render_feature(spec) + result = render_model(spec) assert "`HexColor`" in result assert "hex_color.md" not in result @@ -1028,15 +1068,15 @@ def test_pipe_character_not_escaped_in_backticks(self) -> None: class TestRenderFeatureWithExamples: - """Tests for render_feature with examples support.""" + """Tests for render_model with examples support.""" def test_accepts_examples_parameter(self) -> None: - """render_feature accepts examples parameter.""" + """render_model accepts examples parameter.""" spec = extract_model(SimpleModel) examples = [ExampleRecord(rows=[("name", "test")])] # Should not raise - result = render_feature(spec, examples=examples) + result = render_model(spec, examples=examples) assert "# SimpleModel" in result def test_renders_single_example_without_heading(self) -> None: @@ -1051,7 +1091,7 @@ class ModelWithCount(BaseModel): spec = extract_model(ModelWithCount) examples = [ExampleRecord(rows=[("name", "test"), ("count", 42)])] - result = render_feature(spec, examples=examples) + result = render_model(spec, examples=examples) assert "## Examples" in result assert "| Column | Value |" in result assert "| `name` | `test` |" in result @@ -1067,7 +1107,7 @@ def test_renders_multiple_examples_with_headings(self) -> None: ExampleRecord(rows=[("name", "second")]), ] - result = render_feature(spec, examples=examples) + result = render_model(spec, examples=examples) assert "## Examples" in result assert "### Example 1" in result assert "### Example 2" in result @@ -1097,7 +1137,7 @@ class TestModel(BaseModel): ) ] - result = render_feature(spec, examples=examples) + result = render_model(spec, examples=examples) # String with backticks assert "| `text` | `hello` |" in result # Number with backticks @@ -1110,14 +1150,14 @@ class TestModel(BaseModel): def test_no_examples_omits_section(self) -> None: """When examples is None, Examples section is not rendered.""" spec = extract_model(SimpleModel) - result = render_feature(spec, examples=None) + result = render_model(spec, examples=None) assert "## Examples" not in result def test_empty_examples_list_omits_section(self) -> None: """When examples is empty list, Examples section is not rendered.""" spec = extract_model(SimpleModel) - result = render_feature(spec, examples=[]) + result = render_model(spec, examples=[]) assert "## Examples" not in result @@ -1203,7 +1243,7 @@ def test_shared_fields_have_no_variant_tag(self) -> None: AnnotatedField( field_spec=FieldSpec( name="id", - type_info=STR_TYPE, + shape=STR_TYPE, description="ID", is_required=True, ), @@ -1211,27 +1251,31 @@ def test_shared_fields_have_no_variant_tag(self) -> None: ), ], ) - result = render_feature(spec) + result = render_model(spec) assert "| `id` |" in result assert "*(" not in result # no variant tag def test_variant_fields_have_inline_tag(self) -> None: """Variant-specific fields get *(Variant)* tag.""" + + class RoadSegment(BaseModel): + pass + spec = make_union_spec( name="Segment", annotated_fields=[ AnnotatedField( field_spec=FieldSpec( name="speed_limit", - type_info=STR_TYPE, + shape=STR_TYPE, description=None, is_required=False, ), - variant_sources=("RoadSegment",), + variant_sources=(RoadSegment,), ), ], ) - result = render_feature(spec) + result = render_model(spec) assert "| `speed_limit` *(Road)* |" in result @@ -1283,7 +1327,7 @@ def test_constraint_class_not_linked(self) -> None: assert "[`CountryCodeAlpha2Constraint`](" not in result.display -def _feature_spec() -> object: +def _model_spec() -> object: return extract_model(SimpleModel) @@ -1296,7 +1340,7 @@ def _newtype_spec() -> object: _USED_BY_CASES = [ - pytest.param(_feature_spec, render_feature, id="feature"), + pytest.param(_model_spec, render_model, id="feature"), pytest.param(_enum_spec, render_enum, id="enum"), pytest.param(_newtype_spec, render_newtype, id="newtype"), ] @@ -1334,8 +1378,8 @@ def test_entries_render_without_links_when_no_context( ("spec_factory", "render_fn", "page_path", "expected_link"), [ pytest.param( - _feature_spec, - render_feature, + _model_spec, + render_model, PurePosixPath("types/strings/hex_color.md"), "../../buildings/building/building.md", id="feature", diff --git a/packages/overture-schema-codegen/tests/test_markdown_type_format.py b/packages/overture-schema-codegen/tests/test_markdown_type_format.py index e54426f5f..b10a69451 100644 --- a/packages/overture-schema-codegen/tests/test_markdown_type_format.py +++ b/packages/overture-schema-codegen/tests/test_markdown_type_format.py @@ -2,21 +2,30 @@ from enum import Enum from pathlib import PurePosixPath -from typing import Literal, NewType - -from overture.schema.codegen.extraction.specs import FieldSpec, TypeIdentity -from overture.schema.codegen.extraction.type_analyzer import ( - TypeInfo, - TypeKind, - analyze_type, +from typing import Any, Literal, NewType + +import pytest +from overture.schema.codegen.extraction.field import ( + AnyScalar, + ArrayOf, + FieldShape, + LiteralScalar, + NewTypeShape, + Primitive, + Scalar, + UnionRef, ) +from overture.schema.codegen.extraction.model_extraction import extract_model +from overture.schema.codegen.extraction.specs import FieldSpec, TypeIdentity, UnionSpec +from overture.schema.codegen.extraction.type_analyzer import analyze_type from overture.schema.codegen.markdown.link_computation import LinkContext from overture.schema.codegen.markdown.type_format import ( - format_dict_type, + _bare_map_side_name, + _registry_name, format_type, format_underlying_type, ) -from overture.schema.system.primitive import int32 +from overture.schema.system.primitive import BBox, Geometry, int32 from pydantic import BaseModel, HttpUrl @@ -28,38 +37,46 @@ class _ModelB(BaseModel): y: str +class _OuterWithModelMap(BaseModel): + m: dict[str, _ModelB] + + +class _OuterWithModelListMap(BaseModel): + m: dict[str, list[_ModelB]] + + class TestFormatType: """Tests for format_type.""" def test_plain_str_renders_as_string(self) -> None: - ti = analyze_type(str) - assert format_type(_make_field(ti)) == "`string`" + assert format_type(_make_field(str)) == "`string`" def test_optional_adds_qualifier(self) -> None: - ti = analyze_type(str | None) - assert format_type(_make_field(ti, is_required=False)) == "`string` (optional)" + assert ( + format_type(_make_field(str | None, is_required=False)) + == "`string` (optional)" + ) def test_literal_renders_as_quoted_value(self) -> None: - ti = analyze_type(Literal["places"]) - assert format_type(_make_field(ti)) == '`"places"`' + assert format_type(_make_field(Literal["places"])) == '`"places"`' def test_multi_value_literal_renders_comma_separated(self) -> None: - ti = analyze_type(Literal["a", "b", "c"]) - assert format_type(_make_field(ti)) == '`"a"` \\| `"b"` \\| `"c"`' + assert ( + format_type(_make_field(Literal["a", "b", "c"])) + == '`"a"` \\| `"b"` \\| `"c"`' + ) def test_enum_without_context_renders_as_code(self) -> None: class Color(str, Enum): RED = "red" - ti = analyze_type(Color) - assert format_type(_make_field(ti)) == "`Color`" + assert format_type(_make_field(Color)) == "`Color`" def test_enum_with_link_context(self) -> None: class Color(str, Enum): RED = "red" - ti = analyze_type(Color) - field = _make_field(ti) + field = _make_field(Color) ctx = LinkContext( page_path=PurePosixPath("buildings/building/building.md"), registry={ @@ -69,55 +86,95 @@ class Color(str, Enum): assert format_type(field, ctx) == "[`Color`](../../types/enums/color.md)" def test_list_of_primitives(self) -> None: - ti = analyze_type(list[str]) - assert format_type(_make_field(ti)) == "`list`" + assert format_type(_make_field(list[str])) == "`list`" def test_nested_list_of_primitives(self) -> None: - ti = analyze_type(list[list[str]]) - assert format_type(_make_field(ti)) == "`list>`" + assert format_type(_make_field(list[list[str]])) == "`list>`" def test_registered_primitive_not_linked(self) -> None: - ti = analyze_type(int32) - result = format_type(_make_field(ti)) + result = format_type(_make_field(int32)) assert result == "`int32`" assert "](int32.md)" not in result + def test_geometry_links_to_aggregate_page(self) -> None: + field = _make_field(Geometry) + ctx = LinkContext( + page_path=PurePosixPath("buildings/building/building.md"), + registry={ + TypeIdentity(Geometry, "Geometry"): PurePosixPath( + "system/primitive/geometry.md" + ) + }, + ) + assert ( + format_type(field, ctx) + == "[`geometry`](../../system/primitive/geometry.md)" + ) -class TestFormatDictType: - """Tests for format_dict_type.""" - - def test_simple_dict_renders_as_map(self) -> None: - ti = analyze_type(dict[str, int]) - result = format_dict_type(ti) - assert result == "map" + def test_bbox_links_to_aggregate_page(self) -> None: + field = _make_field(BBox) + ctx = LinkContext( + page_path=PurePosixPath("base/feature/feature.md"), + registry={ + TypeIdentity(BBox, "BBox"): PurePosixPath( + "system/primitive/geometry.md" + ) + }, + ) + assert format_type(field, ctx) == "[`bbox`](../../system/primitive/geometry.md)" - def test_dict_with_newtype_shows_semantic_name(self) -> None: - MyKey = NewType("MyKey", str) - ti = analyze_type(dict[MyKey, int]) - result = format_dict_type(ti) - assert result == "map" + def test_geometry_without_context_renders_plain_code(self) -> None: + assert format_type(_make_field(Geometry)) == "`geometry`" def _make_field( - ti: TypeInfo, *, name: str = "x", is_required: bool = True + annotation: object, + *, + name: str = "x", + is_required: bool = True, + is_optional: bool = False, ) -> FieldSpec: - """Build a FieldSpec for test convenience.""" - return FieldSpec(name=name, type_info=ti, description=None, is_required=is_required) + """Build a FieldSpec from an annotation for test convenience.""" + if isinstance(annotation, (Scalar, ArrayOf, UnionRef)): + shape: FieldShape = annotation # type: ignore[assignment] + else: + shape, resolved_optional, _ = analyze_type(annotation) + is_optional = is_optional or resolved_optional + return FieldSpec( + name=name, + shape=shape, + description=None, + is_required=is_required, + is_optional=is_optional, + ) + + +def _union_ref(members: list[type]) -> UnionRef: + """Build a UnionRef for tests without running through extract_union.""" + union_spec = UnionSpec( + name=members[0].__name__, + description=None, + annotated_fields=[], + members=members, # type: ignore[arg-type] + discriminator_field=None, + discriminator_mapping=None, + source_annotation=object(), + common_base=BaseModel, + ) + return UnionRef(union=union_spec) class TestFormatUnionType: - """Tests for UNION-kind TypeInfo in format_type.""" + """Tests for union FieldShape in format_type.""" def test_union_renders_all_members(self) -> None: - ti = analyze_type(_ModelA | _ModelB) - result = format_type(_make_field(ti)) + result = format_type(_make_field(_union_ref([_ModelA, _ModelB]))) assert "`_ModelA`" in result assert "`_ModelB`" in result # Pipe separator escaped for table cells assert r"\|" in result def test_union_with_link_context_links_each_member(self) -> None: - ti = analyze_type(_ModelA | _ModelB) ctx = LinkContext( page_path=PurePosixPath("theme/feature/feature.md"), registry={ @@ -129,39 +186,36 @@ def test_union_with_link_context_links_each_member(self) -> None: ), }, ) - result = format_type(_make_field(ti), ctx) + result = format_type(_make_field(_union_ref([_ModelA, _ModelB])), ctx) assert "[`_ModelA`](types/model_a.md)" in result assert "[`_ModelB`](types/model_b.md)" in result def test_optional_union_adds_qualifier(self) -> None: - ti = analyze_type(_ModelA | _ModelB | None) - result = format_type(_make_field(ti, is_required=False)) + result = format_type( + _make_field( + _union_ref([_ModelA, _ModelB]), is_required=False, is_optional=True + ) + ) assert "(optional)" in result assert "`_ModelA`" in result assert "`_ModelB`" in result def test_list_of_union_adds_qualifier(self) -> None: - ti = TypeInfo( - base_type="_ModelA", - kind=TypeKind.UNION, - list_depth=1, - union_members=(_ModelA, _ModelB), - ) - result = format_type(_make_field(ti)) + """list[union] renders with (list) qualifier.""" + shape = ArrayOf(element=_union_ref([_ModelA, _ModelB])) + result = format_type(_make_field(shape)) assert "(list)" in result assert "`_ModelA`" in result assert "`_ModelB`" in result def test_union_members_unlinked_without_context(self) -> None: - ti = analyze_type(_ModelA | _ModelB) - result = format_type(_make_field(ti)) + result = format_type(_make_field(_union_ref([_ModelA, _ModelB]))) # No markdown links without context assert "]()" not in result assert "[`" not in result def test_union_partial_links(self) -> None: """Members with pages get linked; members without don't.""" - ti = analyze_type(_ModelA | _ModelB) ctx = LinkContext( page_path=PurePosixPath("theme/feature/feature.md"), registry={ @@ -170,19 +224,45 @@ def test_union_partial_links(self) -> None: ) }, ) - result = format_type(_make_field(ti), ctx) + result = format_type(_make_field(_union_ref([_ModelA, _ModelB])), ctx) assert "[`_ModelA`](types/model_a.md)" in result assert "`_ModelB`" in result # _ModelB should NOT be linked assert "[`_ModelB`]" not in result +class TestScalarVariantRendering: + """format_type and _registry_name handle all three Scalar variants correctly.""" + + def test_registry_name_any_scalar(self) -> None: + assert _registry_name(AnyScalar()) == "Any" + + def test_registry_name_literal_scalar(self) -> None: + assert _registry_name(LiteralScalar(values=("road",))) == "Literal" + + def test_any_scalar_renders_as_Any(self) -> None: + assert format_type(_make_field(AnyScalar())) == "`Any`" + + def test_literal_scalar_renders_first_value_quoted(self) -> None: + # LiteralScalar goes through the Literal path in format_type, not _registry_name + assert format_type(_make_field(LiteralScalar(values=("road",)))) == '`"road"`' + + def test_literal_scalar_multi_value(self) -> None: + result = format_type(_make_field(LiteralScalar(values=("a", "b")))) + assert '`"a"`' in result + assert '`"b"`' in result + + def test_list_of_literal_single_value(self) -> None: + assert format_type(_make_field(list[Literal["road"]])) == '`list<"road">`' + + def test_list_of_literal_multi_value(self) -> None: + assert format_type(_make_field(list[Literal["a", "b"]])) == '`list<"a" | "b">`' + + class TestPydanticTypeLinking: """Tests for PRIMITIVE types with pages getting linked.""" def test_pydantic_type_linked_when_in_registry(self) -> None: - ti = analyze_type(HttpUrl) - field = _make_field(ti) ctx = LinkContext( page_path=PurePosixPath("places/place/place.md"), registry={ @@ -191,24 +271,20 @@ def test_pydantic_type_linked_when_in_registry(self) -> None: ) }, ) - result = format_type(field, ctx) + result = format_type(_make_field(HttpUrl), ctx) assert "[`HttpUrl`]" in result assert "pydantic/networks/http_url.md" in result def test_pydantic_type_unlinked_without_registry_entry(self) -> None: - ti = analyze_type(HttpUrl) - field = _make_field(ti) ctx = LinkContext( page_path=PurePosixPath("places/place/place.md"), registry={}, ) - result = format_type(field, ctx) + result = format_type(_make_field(HttpUrl), ctx) assert result == "`HttpUrl`" assert "[" not in result def test_list_of_pydantic_type_linked(self) -> None: - ti = analyze_type(list[HttpUrl]) - field = _make_field(ti) ctx = LinkContext( page_path=PurePosixPath("places/place/place.md"), registry={ @@ -217,14 +293,12 @@ def test_list_of_pydantic_type_linked(self) -> None: ) }, ) - result = format_type(field, ctx) + result = format_type(_make_field(list[HttpUrl]), ctx) assert "HttpUrl" in result assert "pydantic/networks/http_url.md" in result def test_registered_primitive_links_to_aggregate_page(self) -> None: """int32 links to the primitives aggregate page when in registry.""" - ti = analyze_type(int32) - field = _make_field(ti) ctx = LinkContext( page_path=PurePosixPath("places/place/place.md"), registry={ @@ -233,7 +307,7 @@ def test_registered_primitive_links_to_aggregate_page(self) -> None: ) }, ) - result = format_type(field, ctx) + result = format_type(_make_field(int32), ctx) assert "[`int32`]" in result assert "system/primitive/primitives.md" in result @@ -249,8 +323,7 @@ class TestListOfSemanticNewtype: def test_list_of_scalar_newtype_renders_list_syntax(self) -> None: """list[ScalarNewType] renders as list, not Name (list).""" ScalarNT = NewType("ScalarNT", str) - ti = analyze_type(list[ScalarNT]) - result = format_type(_make_field(ti)) + result = format_type(_make_field(list[ScalarNT])) assert "list<" in result assert "ScalarNT" in result assert "(list)" not in result @@ -258,23 +331,20 @@ def test_list_of_scalar_newtype_renders_list_syntax(self) -> None: def test_newtype_wrapping_list_renders_qualifier(self) -> None: """NewType wrapping list[X] renders as Name (list).""" ListNT = NewType("ListNT", list[str]) - ti = analyze_type(ListNT) - result = format_type(_make_field(ti)) + result = format_type(_make_field(ListNT)) assert "(list)" in result assert "ListNT" in result def test_list_of_scalar_newtype_with_link(self) -> None: """list[ScalarNewType] with link context renders linked list.""" ScalarNT = NewType("ScalarNT", str) - ti = analyze_type(list[ScalarNT]) - field = _make_field(ti) ctx = LinkContext( page_path=PurePosixPath("places/place/place.md"), registry={ TypeIdentity(ScalarNT, "ScalarNT"): PurePosixPath("system/scalar_nt.md") }, ) - result = format_type(field, ctx) + result = format_type(_make_field(list[ScalarNT]), ctx) assert "list<" in result assert "ScalarNT" in result assert "system/scalar_nt.md" in result @@ -283,24 +353,204 @@ def test_list_of_scalar_newtype_with_link(self) -> None: def test_nested_list_of_scalar_newtype_renders_nested_list_syntax(self) -> None: """list[list[ScalarNewType]] renders as list>.""" ScalarNT = NewType("ScalarNT", str) - ti = analyze_type(list[list[ScalarNT]]) - result = format_type(_make_field(ti)) + result = format_type(_make_field(list[list[ScalarNT]])) assert "list<" in result assert "list<`" in result or "`list LinkContext: + """Build a LinkContext registering each (newtype, name, page_path).""" + return LinkContext( + page_path=PurePosixPath("base/names/names.md"), + registry={ + TypeIdentity(newtype, name): PurePosixPath(path) + for newtype, name, path in entries + }, + ) + + +class TestFormatMapType: + """Tests for MapOf rendering in field cells (format_type). + + Adjacent backtick runs break links: CommonMark reads `` as a + multi-backtick code-span delimiter, so any side left bare must fold + into the surrounding `map<...>` span. Every linked case asserts + `"``" not in result` to guard that. + """ + + def test_map_without_context_renders_bare_names(self) -> None: + """A map field with no link context renders bare key/value names.""" + assert format_type(_make_field(dict[str, int32])) == "`map`" + + def test_map_value_list_preserves_list_wrapper(self) -> None: + """A list-valued map keeps its `list<...>` wrapper, not just the element.""" + assert ( + format_type(_make_field(dict[str, list[int]])) + == "`map>`" + ) + + def test_map_value_nested_map_preserves_map_wrapper(self) -> None: + """A map-valued map keeps its inner `map<...>`, not a bare `?`.""" + assert ( + format_type(_make_field(dict[str, dict[str, int]])) + == "`map>`" + ) + + def test_map_value_any_renders_as_any_not_question_mark(self) -> None: + """An `Any`-valued map names the value `Any`, never a bare `?`.""" + assert format_type(_make_field(dict[str, Any])) == "`map`" + + def test_union_valued_map_side_raises(self) -> None: + """A union-valued map side fails loudly rather than rendering a guess.""" + with pytest.raises(NotImplementedError): + _bare_map_side_name(_union_ref([_ModelA, _ModelB])) + + def test_non_semantic_newtype_map_side_uses_registry_name(self) -> None: + """A pass-through NewType resolves to its registry name, not its raw name. + + A NewType whose name equals its base type (here `int`) is not + semantic, so it must render as the registry markdown name (`int64`). + """ + shape = NewTypeShape( + name="int", ref=object(), inner=Primitive(base_type="int", source_type=int) + ) + assert _bare_map_side_name(shape) == "int64" + + def test_map_key_newtype_links_in_field_cell(self) -> None: + """A semantic NewType key links to its page in the field cell.""" + LangTag = NewType("LangTag", str) + ctx = _link_ctx((LangTag, "LangTag", "system/language_tag.md")) + result = format_type(_make_field(dict[LangTag, str]), ctx) + assert "[`LangTag`]" in result + assert "language_tag.md" in result + assert "map<" in result + assert "``" not in result + + def test_map_value_newtype_links_in_field_cell(self) -> None: + """A semantic NewType value links to its page in the field cell.""" + Stripped = NewType("Stripped", str) + ctx = _link_ctx((Stripped, "Stripped", "system/stripped_string.md")) + result = format_type(_make_field(dict[str, Stripped]), ctx) + assert "[`Stripped`]" in result + assert "stripped_string.md" in result + assert "``" not in result + + def test_map_value_model_links_in_field_cell(self) -> None: + """A model-valued map links the value model to its page (bd-ru4n). + + The real pipeline resolves a `dict[K, Model]` value to a `ModelRef` + (not a BaseModel-sourced `Primitive`), so the link path must handle + `ModelRef` map sides, not only `Primitive` ones. + """ + spec = extract_model(_OuterWithModelMap) + field = next(f for f in spec.fields if f.name == "m") + ctx = _link_ctx((_ModelB, "_ModelB", "theme/feature/types/model_b.md")) + result = format_type(field, ctx) + assert "[`_ModelB`]" in result + assert "model_b.md" in result + assert "map<" in result + assert "``" not in result + + def test_map_value_model_list_renders_bare_with_wrapper(self) -> None: + """A `list`-valued map keeps its `list<...>` wrapper, no link. + + The real pipeline resolves the value to `ArrayOf(element=ModelRef)`. + Linking would collapse the wrapper to a bare model link, so the + `depth == 0` guard keeps the model side bare and preserves + `list<...>`. Registering `_ModelB` makes the absent link meaningful. + """ + spec = extract_model(_OuterWithModelListMap) + field = next(f for f in spec.fields if f.name == "m") + ctx = _link_ctx((_ModelB, "_ModelB", "theme/feature/types/model_b.md")) + result = format_type(field, ctx) + assert "[`_ModelB`]" not in result + assert "list<" in result + assert "map>" in result + + def test_map_key_and_value_newtypes_both_link(self) -> None: + """When both sides are semantic NewTypes, both link in the field cell.""" + LangTag = NewType("LangTag", str) + Stripped = NewType("Stripped", str) + ctx = _link_ctx( + (LangTag, "LangTag", "system/language_tag.md"), + (Stripped, "Stripped", "system/stripped_string.md"), + ) + result = format_type(_make_field(dict[LangTag, Stripped]), ctx) + assert "[`LangTag`]" in result + assert "language_tag.md" in result + assert "[`Stripped`]" in result + assert "stripped_string.md" in result + assert "``" not in result + + def test_map_value_geometry_links_in_field_cell(self) -> None: + """A Geometry-valued map links to the geometry page. + + Geometry is a class-based registered primitive, not Enum/BaseModel. + The map-side link decision shares `_scalar_identity`'s coverage, so a + Geometry value links rather than rendering bare. + """ + ctx = _link_ctx((Geometry, "geometry", "system/primitive/geometry.md")) + result = format_type(_make_field(dict[str, Geometry]), ctx) + assert "[`geometry`]" in result + assert "system/primitive/geometry.md" in result + assert "``" not in result + + def test_map_value_pydantic_type_links_in_field_cell(self) -> None: + """A pydantic-sourced map value links to its page. + + HttpUrl is pydantic-sourced, not Enum/BaseModel; the shared map-side + decision links it where the narrow Enum/BaseModel-only check left it + bare. + """ + ctx = _link_ctx((HttpUrl, "HttpUrl", "pydantic/networks/http_url.md")) + result = format_type(_make_field(dict[str, HttpUrl]), ctx) + assert "[`HttpUrl`]" in result + assert "pydantic/networks/http_url.md" in result + assert "``" not in result + + +class TestFormatUnderlyingScalarType: + """Tests for scalar terminals in format_underlying_type.""" + + def test_geometry_underlying_type_links(self) -> None: + """A NewType whose underlying type is Geometry links to its page.""" + shape, _, _ = analyze_type(NewType("GeomAlias", Geometry)) + ctx = LinkContext( + page_path=PurePosixPath("system/types/geom_alias.md"), + registry={ + TypeIdentity(Geometry, "geometry"): PurePosixPath( + "system/primitive/geometry.md" + ) + }, + ) + result = format_underlying_type(shape, ctx) + assert "[`geometry`](../primitive/geometry.md)" in result + + def test_numeric_underlying_type_stays_bare(self) -> None: + """A NewType over a numeric primitive renders bare, not over-linked. + + The numeric branch keys on the builtin (`int`), which has no registry + entry, so the underlying type stays bare and keeps its markdown name. + """ + shape, _, _ = analyze_type(int32) + ctx = LinkContext(page_path=PurePosixPath("system/types/x.md"), registry={}) + result = format_underlying_type(shape, ctx) + assert result == "`int32`" + assert "[" not in result + + class TestFormatUnderlyingUnionType: - """Tests for UNION-kind TypeInfo in format_underlying_type.""" + """Tests for union FieldShape in format_underlying_type.""" def test_union_renders_all_members(self) -> None: - ti = analyze_type(_ModelA | _ModelB) - result = format_underlying_type(ti) + shape = _union_ref([_ModelA, _ModelB]) + result = format_underlying_type(shape) assert result == "`_ModelA` | `_ModelB`" def test_union_with_link_context(self) -> None: - ti = analyze_type(_ModelA | _ModelB) + shape = _union_ref([_ModelA, _ModelB]) ctx = LinkContext( page_path=PurePosixPath("types/my_union.md"), registry={ @@ -312,6 +562,6 @@ def test_union_with_link_context(self) -> None: ), }, ) - result = format_underlying_type(ti, ctx) + result = format_underlying_type(shape, ctx) assert "[`_ModelA`](../theme/feature/types/model_a.md)" in result assert "[`_ModelB`](../theme/feature/types/model_b.md)" in result diff --git a/packages/overture-schema-codegen/tests/test_model_extraction.py b/packages/overture-schema-codegen/tests/test_model_extraction.py new file mode 100644 index 000000000..5ee081459 --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_model_extraction.py @@ -0,0 +1,163 @@ +"""Tests for `extract_model`.""" + +from typing import Annotated, Optional + +from overture.schema.codegen.extraction.field import ( + ArrayOf, + ModelRef, + Primitive, + UnionRef, +) +from overture.schema.codegen.extraction.field_walk import terminal_of +from overture.schema.codegen.extraction.length_constraints import ArrayMinLen +from overture.schema.codegen.extraction.model_extraction import extract_model +from overture.schema.common.scoping.vehicle import VehicleSelector +from pydantic import BaseModel, Field + + +def test_extract_model_populates_union_terminal() -> None: + """`extract_model` resolves UNION terminals to a `UnionRef` carrying a `UnionSpec`.""" + + class Container(BaseModel): + items: list[VehicleSelector] + + spec = extract_model(Container) + items_field = next(f for f in spec.fields if f.name == "items") + + terminal = terminal_of(items_field.shape) + assert isinstance(terminal, UnionRef) + assert terminal.union.discriminator_field == "dimension" + + +def test_required_list_with_optional_element_is_required() -> None: + """A required `list[X | None]` field must not inherit element optionality. + + `list[str | None]` is a list whose elements may be None; the field + itself still requires a list to be present. The list branch must + return `False` for field optionality so `FieldSpec.is_required` stays + `True` and `check_required` is generated for the field. + """ + + class M(BaseModel): + tags: list[str | None] + + spec = extract_model(M) + tags_field = next(f for f in spec.fields if f.name == "tags") + + assert tags_field.is_optional is False + assert tags_field.is_required is True + assert isinstance(tags_field.shape, ArrayOf) + assert isinstance(tags_field.shape.element, Primitive) + assert tags_field.shape.element.base_type == "str" + + +def test_required_list_plain_element_is_required() -> None: + """A required `list[str]` field is unaffected by the optionality fix.""" + + class M(BaseModel): + tags: list[str] + + spec = extract_model(M) + tags_field = next(f for f in spec.fields if f.name == "tags") + + assert tags_field.is_optional is False + assert tags_field.is_required is True + assert isinstance(tags_field.shape, ArrayOf) + + +def test_optional_list_with_optional_element_is_optional() -> None: + """A `list[str | None] | None` field is optional (the outer | None).""" + + class M(BaseModel): + tags: list[str | None] | None = None + + spec = extract_model(M) + tags_field = next(f for f in spec.fields if f.name == "tags") + + assert tags_field.is_optional is True + assert tags_field.is_required is False + + +def test_self_referential_list_forward_ref_resolves_to_cycle() -> None: + """A `list["Self"]` forward ref resolves to a cycle-marked `ModelRef`. + + Builtin generics store `list["Node"]`'s element as a bare `str`, + which neither Pydantic nor `typing.get_type_hints` resolves. + `extract_model` must resolve it against the model's namespace so the + self-reference reaches its model terminal and the cycle guard marks + the back-edge -- rather than crashing the type analyzer's terminal + classifier on an unresolved string. + """ + + class Node(BaseModel): + val: Annotated[int, Field(ge=0)] + children: list["Node"] = Field(default_factory=list) + + spec = extract_model(Node) + children = next(f for f in spec.fields if f.name == "children") + + assert isinstance(children.shape, ArrayOf) + element = children.shape.element + assert isinstance(element, ModelRef) + assert element.starts_cycle is True + assert element.model is spec + + +def test_self_referential_optional_resolves_to_cycle() -> None: + """A top-level `Optional["Self"]` resolves to a cycle-marked `ModelRef`. + + Pydantic resolves the string inside `Optional["Node"]` before + `extract_model` runs -- the annotation's args are already + `(, NoneType)`, so no string reaches `_resolve_forward_ref`. + This exercises the cycle guard through a Pydantic-resolved `Optional`, + not the bare-string forward-ref path covered above. + """ + + class Node(BaseModel): + val: int + parent: Optional["Node"] = None + + spec = extract_model(Node) + parent = next(f for f in spec.fields if f.name == "parent") + + assert isinstance(parent.shape, ModelRef) + assert parent.shape.starts_cycle is True + assert parent.shape.model is spec + + +def test_nested_list_forward_ref_resolves_to_cycle() -> None: + """A `list[list["Self"]]` forward ref resolves through both array layers.""" + + class Node(BaseModel): + val: int + grid: list[list["Node"]] = Field(default_factory=list) + + spec = extract_model(Node) + grid = next(f for f in spec.fields if f.name == "grid") + + assert isinstance(grid.shape, ArrayOf) + assert isinstance(grid.shape.element, ArrayOf) + inner = grid.shape.element.element + assert isinstance(inner, ModelRef) + assert inner.starts_cycle is True + + +def test_field_metadata_minlen_wrapped_as_array_min_len() -> None: + """MinLen in field_info.metadata is wrapped to ArrayMinLen, not left as raw MinLen. + + Pydantic strips the outermost Annotated wrapper from non-optional, + non-union list fields and moves MinLen to field_info.metadata. Without + routing through attach_constraints, the raw MinLen would survive into + the constraint table untyped, causing dispatch to raise TypeError at + codegen time. + """ + + class M(BaseModel): + items: list[str] = Field(min_length=2) + + spec = extract_model(M) + items_field = next(f for f in spec.fields if f.name == "items") + + assert isinstance(items_field.shape, ArrayOf) + constraints = [cs.constraint for cs in items_field.shape.constraints] + assert ArrayMinLen(min_length=2) in constraints diff --git a/packages/overture-schema-codegen/tests/test_model_extractor.py b/packages/overture-schema-codegen/tests/test_model_extractor.py index f2b2bd257..981d7bf78 100644 --- a/packages/overture-schema-codegen/tests/test_model_extractor.py +++ b/packages/overture-schema-codegen/tests/test_model_extractor.py @@ -12,11 +12,13 @@ assert_literal_field, find_field, ) -from overture.schema.codegen.extraction.model_extraction import ( - expand_model_tree, - extract_model, +from overture.schema.codegen.extraction.field import ModelRef, Primitive +from overture.schema.codegen.extraction.field_walk import ( + all_constraints, + has_array_layer, + terminal_of, ) -from overture.schema.codegen.extraction.specs import ModelSpec +from overture.schema.codegen.extraction.model_extraction import extract_model from overture.schema.system.field_constraint import UniqueItemsConstraint from overture.schema.system.model_constraint import ( FieldEqCondition, @@ -90,7 +92,9 @@ class SimpleModel(BaseModel): assert result.description == "A simple test model." assert len(result.fields) == 1 assert result.fields[0].name == "name" - assert result.fields[0].type_info.base_type == "str" + scalar = terminal_of(result.fields[0].shape) + assert isinstance(scalar, Primitive) + assert scalar.base_type == "str" assert result.fields[0].is_required is True def test_extract_model_does_not_set_entry_point(self) -> None: @@ -118,7 +122,7 @@ class ModelWithOptional(BaseModel): nickname_field = find_field(result, "nickname") assert nickname_field.is_required is False - assert nickname_field.type_info.is_optional is True + assert nickname_field.is_optional is True def test_extract_model_with_field_description(self) -> None: """Should extract field descriptions from Field().""" @@ -144,8 +148,10 @@ class ModelWithList(BaseModel): tags_field = result.fields[0] assert tags_field.name == "tags" - assert tags_field.type_info.is_list is True - assert tags_field.type_info.base_type == "str" + assert has_array_layer(tags_field.shape) + scalar = terminal_of(tags_field.shape) + assert isinstance(scalar, Primitive) + assert scalar.base_type == "str" class TestExtractModelWithThemeType: @@ -365,95 +371,91 @@ class Child(Parent, ChildMixin): assert field_names == ["core", "p", "pm", "own", "cm"] -class TestExpandModelTree: - """Tests for expand_model_tree.""" +class TestSubModelExpansion: + """Sub-model resolution at extract_model time.""" def test_model_without_sub_models_unchanged(self) -> None: - """Fields without MODEL kind get model=None.""" + """Fields without MODEL kind have no ModelRef in their shape.""" class Simple(BaseModel): name: str count: int spec = extract_model(Simple) - expand_model_tree(spec) for f in spec.fields: - assert f.model is None - assert f.starts_cycle is False + assert not isinstance(terminal_of(f.shape), ModelRef) def test_nested_model_gets_expanded(self) -> None: - """MODEL-kind fields get their model populated.""" + """MODEL-kind fields resolve to a ModelRef in the shape.""" spec = extract_model(FeatureWithAddress) - expand_model_tree(spec) addr_field = find_field(spec, "address") - assert addr_field.model is not None - assert addr_field.model.name == "Address" - assert addr_field.starts_cycle is False + terminal = terminal_of(addr_field.shape) + assert isinstance(terminal, ModelRef) + assert terminal.model.name == "Address" + assert terminal.starts_cycle is False # Sub-model fields should exist - sub_names = [f.name for f in addr_field.model.fields] + sub_names = [f.name for f in terminal.model.fields] assert "street" in sub_names assert "city" in sub_names def test_cycle_detected_and_marked(self) -> None: - """Self-referential model gets starts_cycle=True.""" + """Self-referential model gets starts_cycle=True on the ModelRef.""" spec = extract_model(TreeNode) - expand_model_tree(spec) parent_field = find_field(spec, "parent") - assert parent_field.model is not None - assert parent_field.model is spec # Same object -- cycle - assert parent_field.starts_cycle is True + terminal = terminal_of(parent_field.shape) + assert isinstance(terminal, ModelRef) + assert terminal.model is spec # Same object -- cycle + assert terminal.starts_cycle is True - def test_shared_reference_not_marked_as_cycle(self) -> None: - """Two models referencing the same sub-model share it without cycle.""" + def test_shared_reference_within_one_extraction(self) -> None: + """Two fields referencing the same sub-model share the RecordSpec.""" class Shared(BaseModel): value: str - class ModelA(BaseModel): - ref: Shared + class Container(BaseModel): + first: Shared + second: Shared - class ModelB(BaseModel): - ref: Shared + spec = extract_model(Container) + first = find_field(spec, "first") + second = find_field(spec, "second") - cache: dict[type, ModelSpec] = {} - spec_a = extract_model(ModelA) - expand_model_tree(spec_a, cache) - - spec_b = extract_model(ModelB) - expand_model_tree(spec_b, cache) - - ref_a = find_field(spec_a, "ref") - ref_b = find_field(spec_b, "ref") - - # Same ModelSpec object, neither is a cycle - assert ref_a.model is ref_b.model - assert ref_a.starts_cycle is False - assert ref_b.starts_cycle is False + first_ref = terminal_of(first.shape) + second_ref = terminal_of(second.shape) + assert isinstance(first_ref, ModelRef) + assert isinstance(second_ref, ModelRef) + # Within one extract_model call, the cache ensures the same + # RecordSpec is reused for both references; neither is a cycle. + assert first_ref.model is second_ref.model + assert first_ref.starts_cycle is False + assert second_ref.starts_cycle is False def test_list_of_model_gets_expanded(self) -> None: - """list[Model] fields also get their model populated.""" + """list[Model] fields also get their model populated via ModelRef.""" class HasList(BaseModel): items: list[SourceItem] spec = extract_model(HasList) - expand_model_tree(spec) items_field = find_field(spec, "items") - assert items_field.model is not None - assert items_field.model.name == "SourceItem" + terminal = terminal_of(items_field.shape) + assert isinstance(terminal, ModelRef) + assert terminal.model.name == "SourceItem" class TestFieldInfoMetadataConstraints: - """Constraints from field_info.metadata are merged into TypeInfo. + """Constraints from `field_info.metadata` attach to the field's shape. Pydantic strips the Annotated wrapper from some fields and moves the - metadata to field_info.metadata. extract_model merges these back into - TypeInfo.constraints so they aren't silently dropped. + metadata to `field_info.metadata`. `extract_model` attaches these + constraints to the appropriate `FieldShape` layer so they aren't + silently dropped. """ def test_geometry_type_constraint_extracted(self) -> None: @@ -462,7 +464,7 @@ def test_geometry_type_constraint_extracted(self) -> None: geometry_field = find_field(spec, "geometry") constraint_types = [ - type(cs.constraint) for cs in geometry_field.type_info.constraints + type(cs.constraint) for cs in all_constraints(geometry_field.shape) ] assert GeometryTypeConstraint in constraint_types @@ -473,7 +475,7 @@ def test_geometry_type_constraint_has_null_source(self) -> None: geo_constraints = [ cs - for cs in geometry_field.type_info.constraints + for cs in all_constraints(geometry_field.shape) if isinstance(cs.constraint, GeometryTypeConstraint) ] assert len(geo_constraints) == 1 @@ -490,7 +492,7 @@ def test_metadata_constraints_not_duplicated(self) -> None: unique_constraints = [ cs - for cs in tags_field.type_info.constraints + for cs in all_constraints(tags_field.shape) if isinstance(cs.constraint, UniqueItemsConstraint) ] assert len(unique_constraints) == 1 @@ -509,7 +511,7 @@ class Model(BaseModel): geo_field = find_field(spec, "geo") constraint_types = [ - type(cs.constraint) for cs in geo_field.type_info.constraints + type(cs.constraint) for cs in all_constraints(geo_field.shape) ] assert GeometryTypeConstraint in constraint_types diff --git a/packages/overture-schema-codegen/tests/test_newtype_extraction.py b/packages/overture-schema-codegen/tests/test_newtype_extraction.py index 6cd73c5c2..9f28dce7e 100644 --- a/packages/overture-schema-codegen/tests/test_newtype_extraction.py +++ b/packages/overture-schema-codegen/tests/test_newtype_extraction.py @@ -3,6 +3,8 @@ from typing import Annotated, NewType from codegen_test_support import STR_TYPE +from overture.schema.codegen.extraction.field import ArrayOf, Primitive +from overture.schema.codegen.extraction.field_walk import terminal_scalar from overture.schema.codegen.extraction.newtype_extraction import extract_newtype from overture.schema.codegen.extraction.specs import NewTypeSpec from overture.schema.system.field_constraint import UniqueItemsConstraint @@ -19,15 +21,19 @@ def test_extract_hex_color(self) -> None: spec = extract_newtype(HexColor) assert spec.name == "HexColor" - assert spec.type_info.newtype_name == "HexColor" + # Outermost NewTypeShape stripped; shape is the underlying scalar. + assert terminal_scalar(spec.shape) is not None def test_extract_id(self) -> None: """Should extract Id NewType with nested chain.""" spec = extract_newtype(Id) assert spec.name == "Id" - assert spec.type_info.newtype_name == "Id" - assert spec.type_info.base_type == "NoWhitespaceString" + # Id wraps NoWhitespaceString, which is a registered semantic newtype + # resolving to a Scalar. After stripping "Id", shape is Scalar with + # base_type "NoWhitespaceString". + assert isinstance(spec.shape, Primitive) + assert spec.shape.base_type == "NoWhitespaceString" def test_extract_newtype_wrapping_list(self) -> None: """Should extract a list-wrapping NewType.""" @@ -41,8 +47,8 @@ class Item(BaseModel): spec = extract_newtype(TestSources) assert spec.name == "TestSources" - assert spec.type_info.is_list is True - assert spec.type_info.newtype_name == "TestSources" + # After stripping the outer NewTypeShape("TestSources"), shape is ArrayOf. + assert isinstance(spec.shape, ArrayOf) def test_extract_newtype_without_doc_uses_field_description(self) -> None: """NewType with Field(description=...) but no __doc__ uses Field description.""" @@ -66,7 +72,7 @@ class TestNewTypeSpecSourceType: """Tests for source_type on NewTypeSpec.""" def test_newtype_spec_source_type_defaults_to_none(self) -> None: - spec = NewTypeSpec(name="Test", description=None, type_info=STR_TYPE) + spec = NewTypeSpec(name="Test", description=None, shape=STR_TYPE) assert spec.source_type is None def test_extract_newtype_sets_source_type(self) -> None: diff --git a/packages/overture-schema-codegen/tests/test_numeric_extraction.py b/packages/overture-schema-codegen/tests/test_numeric_extraction.py index ee604ba75..6f3a5767f 100644 --- a/packages/overture-schema-codegen/tests/test_numeric_extraction.py +++ b/packages/overture-schema-codegen/tests/test_numeric_extraction.py @@ -55,7 +55,7 @@ class TestExtractNumericBounds: def test_signed_integer_bounds(self) -> None: """Should extract ge/le from a constrained integer NewType.""" spec = extract_newtype(int32) - bounds = extract_numeric_bounds(spec.type_info) + bounds = extract_numeric_bounds(spec.shape) assert bounds.ge == -(2**31) assert bounds.le == 2**31 - 1 @@ -63,7 +63,7 @@ def test_signed_integer_bounds(self) -> None: def test_unsigned_integer_bounds(self) -> None: """Should extract 0-based bounds from unsigned NewType.""" spec = extract_newtype(uint8) - bounds = extract_numeric_bounds(spec.type_info) + bounds = extract_numeric_bounds(spec.shape) assert bounds.ge == 0 assert bounds.le == 255 @@ -71,7 +71,7 @@ def test_unsigned_integer_bounds(self) -> None: def test_int64_bounds(self) -> None: """Should extract large bounds from int64.""" spec = extract_newtype(int64) - bounds = extract_numeric_bounds(spec.type_info) + bounds = extract_numeric_bounds(spec.shape) assert bounds.ge == -(2**63) assert bounds.le == 2**63 - 1 @@ -79,7 +79,7 @@ def test_int64_bounds(self) -> None: def test_unconstrained_type(self) -> None: """Should return empty Interval for types without numeric constraints.""" spec = extract_newtype(float32) - bounds = extract_numeric_bounds(spec.type_info) + bounds = extract_numeric_bounds(spec.shape) assert bounds.ge is None assert bounds.gt is None @@ -91,8 +91,8 @@ def test_exclusive_bounds(self) -> None: ExclusiveBounded = NewType( "ExclusiveBounded", Annotated[int, Field(gt=0, lt=100)] ) - type_info = analyze_type(ExclusiveBounded) - bounds = extract_numeric_bounds(type_info) + shape, _, _ = analyze_type(ExclusiveBounded) + bounds = extract_numeric_bounds(shape) assert bounds.gt == 0 assert bounds.lt == 100 @@ -102,8 +102,8 @@ def test_exclusive_bounds(self) -> None: def test_mixed_bounds(self) -> None: """Should extract a mix of inclusive and exclusive bounds.""" MixedBounded = NewType("MixedBounded", Annotated[int, Field(ge=0, lt=256)]) - type_info = analyze_type(MixedBounded) - bounds = extract_numeric_bounds(type_info) + shape, _, _ = analyze_type(MixedBounded) + bounds = extract_numeric_bounds(shape) assert bounds.ge == 0 assert bounds.lt == 256 diff --git a/packages/overture-schema-codegen/tests/test_pyspark_base_row.py b/packages/overture-schema-codegen/tests/test_pyspark_base_row.py new file mode 100644 index 000000000..a1e9ffb94 --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_pyspark_base_row.py @@ -0,0 +1,589 @@ +"""Tests for valid-row generation from ModelSpecs.""" + +import uuid +from enum import Enum +from typing import Any + +import pytest +from annotated_types import Gt, Lt +from codegen_test_support import ( + FeatureWithDict, + FeatureWithRequiredUrl, + RequireAnyTrueModel, + discover_feature, + spec_for_model, +) +from overture.schema.codegen.extraction.field import ( + AnyScalar, + ConstraintSource, + LiteralScalar, + ModelRef, + Primitive, +) +from overture.schema.codegen.extraction.field_walk import terminal_of +from overture.schema.codegen.extraction.model_extraction import extract_model +from overture.schema.codegen.extraction.specs import ( + FieldSpec, + ModelSpec, + UnionSpec, +) +from overture.schema.codegen.pyspark.constraint_dispatch import ExpressionDescriptor +from overture.schema.codegen.pyspark.test_data.base_row import ( + _primitive_default, + _row_satisfies_condition, + _satisfy_model_constraints, + _value_from_check_pattern, + _value_from_scalar_constraints, + condition_overrides_for_present_field, + generate_arm_rows, + generate_base_row, + generate_populated_arm_rows, + generate_populated_row, + value_for_field, +) +from overture.schema.system.model_constraint import ( + FieldEqCondition, + forbid_if, + min_fields_set, + require_if, +) +from pydantic import BaseModel, Field, HttpUrl, TypeAdapter + + +@pytest.fixture(scope="module") +def connector_spec() -> ModelSpec: + return discover_feature("Connector") + + +@pytest.fixture(scope="module") +def segment_spec() -> ModelSpec: + return discover_feature("Segment") + + +@pytest.fixture(scope="module") +def segment_union(segment_spec: ModelSpec) -> UnionSpec: + assert isinstance(segment_spec, UnionSpec) + return segment_spec + + +class TestRequireAnyTrueBaseRow: + """A require_any_true base row must satisfy at least one condition. + + Both condition fields are optional, so an unadjusted sparse row leaves + them absent -- which the constraint rejects. `_satisfy_model_constraints` + must set one true. + """ + + def test_base_row_satisfies_constraint(self) -> None: + row = generate_base_row(spec_for_model(RequireAnyTrueModel)) + assert row.get("is_land") is True or row.get("is_territorial") is True + + def test_base_row_passes_pydantic_validation(self) -> None: + row = generate_base_row(spec_for_model(RequireAnyTrueModel)) + RequireAnyTrueModel(**row) + + +class TestPrimitiveDefault: + """Primitive defaults for string-like types that need valid placeholders.""" + + def test_http_url_is_valid(self) -> None: + val = _primitive_default("HttpUrl") + TypeAdapter(HttpUrl).validate_python(val) + + def test_email_str_contains_at(self) -> None: + val = _primitive_default("EmailStr") + assert isinstance(val, str) + assert "@" in val + + +class TestBaseRowUrlFields: + """Base rows with URL-typed fields produce Pydantic-valid values.""" + + def test_required_url_field_passes_validation(self) -> None: + spec = spec_for_model(FeatureWithRequiredUrl) + row = generate_base_row(spec) + TypeAdapter(FeatureWithRequiredUrl).validate_python(row) + + +class TestGenerateBaseRow: + def test_passes_pydantic_validation(self, connector_spec: ModelSpec) -> None: + row = generate_base_row(connector_spec) + assert connector_spec.source_type is not None + TypeAdapter(connector_spec.source_type).validate_python(row) + + def test_required_fields_present(self, connector_spec: ModelSpec) -> None: + row = generate_base_row(connector_spec) + required_names = {f.name for f in connector_spec.fields if f.is_required} + assert required_names <= set(row.keys()) + + def test_optional_fields_absent(self, connector_spec: ModelSpec) -> None: + row = generate_base_row(connector_spec) + optional_names = {f.name for f in connector_spec.fields if not f.is_required} + assert optional_names.isdisjoint(set(row.keys())) + + def test_id_is_deterministic_uuid(self, connector_spec: ModelSpec) -> None: + row = generate_base_row(connector_spec) + assert "id" in row + parsed = uuid.UUID(row["id"]) + assert parsed.version == 5 + + def test_geometry_is_valid_wkt(self, connector_spec: ModelSpec) -> None: + row = generate_base_row(connector_spec) + assert "geometry" in row + assert row["geometry"].startswith("POINT") + + +class TestGenerateArmRows: + def test_returns_dict_per_arm( + self, segment_spec: ModelSpec, segment_union: UnionSpec + ) -> None: + rows = generate_arm_rows(segment_spec) + assert segment_union.discriminator_mapping is not None + assert set(rows.keys()) == set(segment_union.discriminator_mapping.keys()) + + def test_each_row_passes_validation( + self, segment_spec: ModelSpec, segment_union: UnionSpec + ) -> None: + rows = generate_arm_rows(segment_spec) + adapter: TypeAdapter[object] = TypeAdapter(segment_union.source_annotation) + for _arm_val, row in rows.items(): + adapter.validate_python(row) + + def test_discriminator_field_set( + self, segment_spec: ModelSpec, segment_union: UnionSpec + ) -> None: + rows = generate_arm_rows(segment_spec) + assert segment_union.discriminator_field is not None + for arm_val, row in rows.items(): + assert row[segment_union.discriminator_field] == arm_val + + def test_arm_specific_required_fields_present( + self, segment_spec: ModelSpec + ) -> None: + """Road arm requires 'class' field; water arm does not.""" + rows = generate_arm_rows(segment_spec) + assert "class" in rows["road"] + assert "class" not in rows["water"] + + +class TestPopulateOptionalFlag: + """populate_optional flag controls recursion depth.""" + + def test_value_for_field_default_skips_optional_children( + self, connector_spec: ModelSpec + ) -> None: + """Default (`populate_optional=False`) yields sparse sub-models.""" + field = next(f for f in connector_spec.fields if f.name == "sources") + model_ref = _list_of_model(field.shape) + val = value_for_field(field, "Connector") + assert isinstance(val, list) + elem = val[0] + assert isinstance(elem, dict) + optional_names = {f.name for f in model_ref.model.fields if not f.is_required} + assert not (optional_names & set(elem.keys())) + + def test_value_for_field_populate_includes_optional_children( + self, connector_spec: ModelSpec + ) -> None: + """`populate_optional=True` yields sub-models that include optional fields.""" + field = next(f for f in connector_spec.fields if f.name == "sources") + model_ref = _list_of_model(field.shape) + val = value_for_field(field, "Connector", populate_optional=True) + assert isinstance(val, list) + elem = val[0] + assert isinstance(elem, dict) + optional_names = {f.name for f in model_ref.model.fields if not f.is_required} + assert optional_names & set(elem.keys()) == optional_names + + +def _list_of_model(shape: object) -> ModelRef: + """Peel `ArrayOf` / `NewTypeShape` layers to reach the inner `ModelRef`.""" + terminal = terminal_of(shape) # type: ignore[arg-type] + assert isinstance(terminal, ModelRef), ( + f"Expected ModelRef terminal, got {type(terminal).__name__}" + ) + return terminal + + +class TestGeneratePopulatedRow: + def test_passes_pydantic_validation(self, connector_spec: ModelSpec) -> None: + row = generate_populated_row(connector_spec) + assert connector_spec.source_type is not None + TypeAdapter(connector_spec.source_type).validate_python(row) + + def test_required_fields_present(self, connector_spec: ModelSpec) -> None: + row = generate_populated_row(connector_spec) + required_names = {f.name for f in connector_spec.fields if f.is_required} + assert required_names <= set(row.keys()) + + def test_optional_fields_present(self, connector_spec: ModelSpec) -> None: + row = generate_populated_row(connector_spec) + optional_names = {f.name for f in connector_spec.fields if not f.is_required} + assert optional_names <= set(row.keys()) + + def test_id_matches_sparse_row(self, connector_spec: ModelSpec) -> None: + sparse = generate_base_row(connector_spec) + populated = generate_populated_row(connector_spec) + assert populated["id"] == sparse["id"] + + def test_nested_structs_populated(self, connector_spec: ModelSpec) -> None: + """Optional struct fields contain populated sub-dicts, not empty.""" + row = generate_populated_row(connector_spec) + assert "sources" in row + elem = row["sources"][0] + sources_field = next(f for f in connector_spec.fields if f.name == "sources") + model_ref = _list_of_model(sources_field.shape) + optional_source_fields = { + f.name for f in model_ref.model.fields if not f.is_required + } + present = optional_source_fields & set(elem.keys()) + assert present == optional_source_fields + + +class TestGeneratePopulatedArmRows: + def test_returns_dict_per_arm( + self, segment_spec: ModelSpec, segment_union: UnionSpec + ) -> None: + rows = generate_populated_arm_rows(segment_spec) + assert segment_union.discriminator_mapping is not None + assert set(rows.keys()) == set(segment_union.discriminator_mapping.keys()) + + def test_each_row_passes_validation( + self, segment_spec: ModelSpec, segment_union: UnionSpec + ) -> None: + rows = generate_populated_arm_rows(segment_spec) + adapter: TypeAdapter[object] = TypeAdapter(segment_union.source_annotation) + for _arm_val, row in rows.items(): + adapter.validate_python(row) + + def test_discriminator_field_set( + self, segment_spec: ModelSpec, segment_union: UnionSpec + ) -> None: + rows = generate_populated_arm_rows(segment_spec) + assert segment_union.discriminator_field is not None + for arm_val, row in rows.items(): + assert row[segment_union.discriminator_field] == arm_val + + def test_optional_fields_present(self, segment_spec: ModelSpec) -> None: + """Populated arm rows include optional fields.""" + rows = generate_populated_arm_rows(segment_spec) + # Road arm has optional speed_limits + road_row = rows["road"] + assert "speed_limits" in road_row + + +class TestMapFieldPopulation: + """MapOf fields are populated with a constraint-valid entry, not `{}`. + + An empty map satisfies Pydantic but leaves nothing for a conformance + scenario to corrupt, so the generated key/value checks would never + fire. The entry's key and value must satisfy their own constraints + (`dict[LanguageTag, StrippedString]` -> key `"en"`, value `"clean"`). + """ + + def test_required_map_field_is_populated(self) -> None: + spec = spec_for_model(FeatureWithDict) + row = generate_base_row(spec) + # metadata: dict[str, int] is required. + assert row["metadata"], "required map field generated as empty dict" + ((k, v),) = row["metadata"].items() + assert isinstance(k, str) + assert isinstance(v, int) + + def test_constrained_map_entry_is_valid(self) -> None: + spec = spec_for_model(FeatureWithDict) + row = generate_populated_row(spec) + assert row["names"], "constrained map field generated as empty dict" + ((key, value),) = row["names"].items() + assert key == "en" # LanguageTagConstraint.valid + assert value == "clean" # StrippedConstraint.valid + + def test_populated_row_passes_pydantic(self) -> None: + spec = spec_for_model(FeatureWithDict) + row = generate_populated_row(spec) + TypeAdapter(FeatureWithDict).validate_python(row) + + def test_any_valued_map_generates_empty(self) -> None: + # `dict[str, Any]` (e.g. Infrastructure.source_tags) has no value + # constraint -- hence no value check -- and `Any` has no value + # strategy, so the map stays empty rather than crashing. + class TagsModel(BaseModel): + source_tags: dict[str, Any] | None = None + + spec = extract_model(TagsModel) + row = generate_populated_row(spec) + assert row.get("source_tags") == {} + + +class TestRawPatternFailsLoud: + """An uncurated raw `Field(pattern=)` fails loud during base-row generation. + + Symmetric with `invalid_value`: both sides point at the missing + `PATTERN_VALUES` entry. Generating a valid value for a pattern with no + curated entry raises an actionable error that names the gap, rather than + yielding a value that fails the pattern and surfaces downstream as a + misleading "row should be valid" Pydantic error. + """ + + _DUMMY_SCALAR = Primitive(base_type="str") + _DUMMY_CS = ConstraintSource(source_ref=None, source_name=None, constraint=object()) + + def test_curated_pattern_returns_valid(self) -> None: + # Sources.license_priority key pattern, anchor-normalized. + desc = ExpressionDescriptor( + function="check_pattern", args=(r"^[A-Za-z0-9._+\-]+\z",) + ) + assert ( + _value_from_check_pattern(desc, self._DUMMY_SCALAR, self._DUMMY_CS) + == "ODbL-1.0" + ) + + def test_uncurated_pattern_raises(self) -> None: + desc = ExpressionDescriptor(function="check_pattern", args=(r"^xyz\z",)) + with pytest.raises(ValueError, match="check_pattern"): + _value_from_check_pattern(desc, self._DUMMY_SCALAR, self._DUMMY_CS) + + def test_uncurated_pattern_field_raises_during_generation(self) -> None: + # End to end: a scalar field carrying an uncurated raw pattern raises + # the actionable error at base-row generation, not as a downstream + # Pydantic "row should be valid" failure. + (meta,) = Field(pattern=r"^[0-9]{4}$").metadata + field = FieldSpec( + name="code", + shape=Primitive( + base_type="str", + constraints=( + ConstraintSource( + source_ref=None, source_name=None, constraint=meta + ), + ), + ), + ) + with pytest.raises(ValueError, match="check_pattern"): + value_for_field(field, "Foo") + + +class TestValueForShapeScalarVariants: + """_value_for_shape handles the Scalar variants it can reach.""" + + def test_any_scalar_raises(self) -> None: + # No schema declares a `dict[K, Any]` value, so `AnyScalar` has no + # value strategy; reaching it raises rather than guessing. + field = FieldSpec(name="x", shape=AnyScalar()) + with pytest.raises(TypeError, match="AnyScalar reached base-row generation"): + value_for_field(field, "Foo") + + def test_literal_scalar_returns_first_value(self) -> None: + field = FieldSpec(name="x", shape=LiteralScalar(values=("road",))) + assert value_for_field(field, "Foo") == "road" + + +class TestMinFieldsSetSatisfied: + """`_satisfy_model_constraints` populates optional fields for `min_fields_set`.""" + + def test_min_fields_set_populates_optional_fields(self) -> None: + @min_fields_set(2) + class MinTwoModel(BaseModel): + a: str | None = None + b: str | None = None + c: str | None = None + + spec = extract_model(MinTwoModel) + row = generate_base_row(spec) + present = [name for name in ("a", "b", "c") if name in row] + assert len(present) >= 2 + + def test_min_fields_set_counts_required_fields(self) -> None: + # Required fields are always present in the sparse base row, and they + # count against `min_fields_set(N)` -- matching Pydantic's + # `model_fields_set` semantics. With one required + three optional + # and `min_fields_set(2)`, the required field plus one optional + # already satisfy the constraint, so the sparse row only needs + # one additional optional fill. + @min_fields_set(2) + class MixedMinModel(BaseModel): + required_field: str + opt_a: str | None = None + opt_b: str | None = None + opt_c: str | None = None + + spec = extract_model(MixedMinModel) + row = generate_base_row(spec) + assert "required_field" in row + present_optional = [n for n in ("opt_a", "opt_b", "opt_c") if n in row] + assert len(present_optional) >= 1 + assert ( + sum( + 1 + for name in row + if name in {"required_field", "opt_a", "opt_b", "opt_c"} + ) + >= 2 + ) + + def test_min_fields_set_all_required_needs_no_optional_fill(self) -> None: + # When required fields alone satisfy `count`, no optional fills are + # needed -- matching Pydantic, which counts required fields toward + # `model_fields_set`. + @min_fields_set(2) + class AllRequiredModel(BaseModel): + req_a: str + req_b: str + opt_a: str | None = None + + spec = extract_model(AllRequiredModel) + row = generate_base_row(spec) + assert "req_a" in row and "req_b" in row + assert "opt_a" not in row + + +class _ModeColor(str, Enum): + RED = "red" + BLUE = "blue" + + +@require_if(["extra"], ~FieldEqCondition("mode", _ModeColor.BLUE)) +class _ModeModelRequireIf(BaseModel): + mode: _ModeColor = _ModeColor.BLUE + extra: str | None = None + + +@forbid_if(["extra"], ~FieldEqCondition("mode", _ModeColor.BLUE)) +class _ModeModelForbidIf(BaseModel): + mode: _ModeColor = _ModeColor.BLUE + extra: str | None = None + + +# Default mode=RED means Not(mode == BLUE) is True from the start, so +# generate_base_row must fill 'extra' without any manual row mutation. +@require_if(["extra"], ~FieldEqCondition("mode", _ModeColor.BLUE)) +class _ModeModelRequireIfTriggered(BaseModel): + mode: _ModeColor = _ModeColor.RED + extra: str | None = None + + +class TestNotConditionBaseRow: + """Base-row generation handles Not(FieldEqCondition) in require_if/forbid_if.""" + + def test_require_if_not_condition_fills_field(self) -> None: + """generate_base_row fills the require_if target when Not-condition holds. + + _ModeModelRequireIfTriggered defaults mode=RED, so Not(mode == BLUE) is + True from the start. generate_base_row must fill 'extra' end-to-end + without any manual row mutation. + """ + spec = extract_model(_ModeModelRequireIfTriggered) + row = generate_base_row(spec) + # 'mode' has a default (RED) and may be omitted from the sparse row; + # what matters is that the Not-condition was evaluated and 'extra' filled. + assert "extra" in row + assert row["extra"] is not None + TypeAdapter(_ModeModelRequireIfTriggered).validate_python(row) + + def test_forbid_if_not_condition_removes_field(self) -> None: + """forbid_if triggered by Not(FieldEqCondition) removes the forbidden field.""" + spec = extract_model(_ModeModelForbidIf) + row: dict[str, object] = { + "mode": _ModeColor.RED.value, + "extra": "should be removed", + } + _satisfy_model_constraints(row, spec) + # With mode='red', Not(mode == BLUE) is True -> extra must be absent + assert "extra" not in row + + def test_unknown_condition_type_raises(self) -> None: + """_row_satisfies_condition must raise for unknown condition kinds.""" + + class _Unknown: + pass + + with pytest.raises((TypeError, NotImplementedError)): + _row_satisfies_condition({}, _Unknown()) + + def test_not_field_eq_condition_base_row_passes_pydantic(self) -> None: + """A model with Not(FieldEqCondition) conditions produces a valid base row.""" + spec = extract_model(_ModeModelRequireIf) + row = generate_base_row(spec) + TypeAdapter(_ModeModelRequireIf).validate_python(row) + + +class _Subtype(str, Enum): + COUNTRY = "country" + REGION = "region" + + +_IS_COUNTRY = FieldEqCondition("subtype", _Subtype.COUNTRY) + + +# Mirrors Division/DivisionBoundary: the base row's first enum member +# (`country`) triggers the forbid, and the symmetric require_if mandates the +# field once the condition is disabled. +@forbid_if(["parent"], _IS_COUNTRY) +@require_if(["parent"], ~_IS_COUNTRY) +class _DivisionLike(BaseModel): + subtype: _Subtype + parent: str | None = None + + +class _Flag(str, Enum): + A = "a" + B = "b" + + +# Disabling the forbid (mode != A) activates a require_if for a field the base +# row lacks, so the override must re-satisfy constraints and carry that field. +@forbid_if(["forbidden"], FieldEqCondition("mode", _Flag.A)) +@require_if(["needed"], FieldEqCondition("mode", _Flag.B)) +class _FlipActivatesRequire(BaseModel): + mode: _Flag + forbidden: str | None = None + needed: str | None = None + + +class TestConditionOverridesForPresentField: + """A forbid_if the base row triggers is disabled so its field can be set.""" + + def test_flips_condition_field_to_disable_forbid(self) -> None: + """The override sets the condition field to a value the forbid rejects.""" + spec = extract_model(_DivisionLike) + overrides = condition_overrides_for_present_field(spec, "parent") + assert overrides == {"subtype": "region"} + + def test_merged_row_with_present_field_is_valid(self) -> None: + """Base row + override + the forbidden field validates against Pydantic.""" + spec = extract_model(_DivisionLike) + overrides = condition_overrides_for_present_field(spec, "parent") + row = {**generate_base_row(spec), **overrides, "parent": "a"} + TypeAdapter(_DivisionLike).validate_python(row) + + def test_field_without_forbid_if_needs_no_override(self) -> None: + """A field no forbid_if gates returns an empty override.""" + spec = extract_model(_DivisionLike) + assert condition_overrides_for_present_field(spec, "subtype") == {} + + def test_resatisfies_constraints_newly_required_by_the_flip(self) -> None: + """Disabling the forbid can activate a require_if; the override fills it.""" + spec = extract_model(_FlipActivatesRequire) + overrides = condition_overrides_for_present_field(spec, "forbidden") + assert overrides["mode"] == "b" + assert overrides.get("needed") is not None + row = {**generate_base_row(spec), **overrides, "forbidden": "x"} + TypeAdapter(_FlipActivatesRequire).validate_python(row) + + +class TestMultiBoundScalarConstraints: + """_value_from_scalar_constraints merges multiple check_bounds before calling valid_bound.""" + + def test_gt_and_lt_float_tight_interval_returns_interior_value(self) -> None: + """Synthesized value satisfies both Gt(0.0) and Lt(1.0) simultaneously.""" + # float bounds: Gt(0.0)+Lt(1.0): gt+1 = 1.0 violates lt=1.0 (boundary) + scalar = Primitive( + base_type="float64", + constraints=( + ConstraintSource(source_ref=None, source_name=None, constraint=Gt(0.0)), + ConstraintSource(source_ref=None, source_name=None, constraint=Lt(1.0)), + ), + ) + result = _value_from_scalar_constraints(scalar) + assert isinstance(result, float) + assert 0.0 < result < 1.0 diff --git a/packages/overture-schema-codegen/tests/test_pyspark_check_builder.py b/packages/overture-schema-codegen/tests/test_pyspark_check_builder.py new file mode 100644 index 000000000..5fa49c72b --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_pyspark_check_builder.py @@ -0,0 +1,2362 @@ +"""Tests for check_builder -- scalar fields, struct recursion, and model constraints.""" + +from dataclasses import replace +from enum import Enum +from typing import Annotated, Literal, NewType, Union + +import pytest +from annotated_types import Ge, Le, MinLen +from codegen_test_support import ( + FeatureWithDict, + LiteralSubtypeModel, + RadioModel, + RequireAnyModel, + TripleNestedArrayModel, + discover_feature, + spec_for_model, + union_spec_for, +) +from overture.schema.codegen.extraction.field import ( + ArrayOf, + ConstraintSource, + FieldShape, + MapOf, + Primitive, + UnionRef, +) +from overture.schema.codegen.extraction.specs import ( + FieldSpec, + ModelSpec, + RecordSpec, +) +from overture.schema.codegen.extraction.union_extraction import extract_union +from overture.schema.codegen.pyspark._render_common import column_level_suffix +from overture.schema.codegen.pyspark.check_builder import ( + build_checks, + classify_map_projection, +) +from overture.schema.codegen.pyspark.check_ir import ( + Check, + ColumnGuard, + ElementGuard, + ModelCheck, +) +from overture.schema.codegen.pyspark.constraint_dispatch import ( + ExpressionDescriptor, + ForbidIf, + RequireIf, + model_constraint_function, +) +from overture.schema.common.scoping.lr import LinearlyReferencedRange +from overture.schema.system.field_constraint.collection import UniqueItemsConstraint +from overture.schema.system.field_path import ( + ArrayPath, + ArraySegment, + FieldPath, + MapPath, + MapProjection, + ScalarPath, + parse, +) +from overture.schema.system.model_constraint import ( + FieldEqCondition, + Not, + forbid_if, + require_any_of, +) +from overture.schema.system.string import CountryCodeAlpha2 +from pydantic import BaseModel, Field +from pydantic.fields import FieldInfo +from pydantic.networks import HttpUrl + +_path = parse + + +def _column_guard(check: Check) -> ColumnGuard | None: + """Return the first ColumnGuard, or None.""" + for g in check.guards: + if isinstance(g, ColumnGuard): + return g + return None + + +def _element_guard(check: Check) -> ElementGuard | None: + """Return the first ElementGuard, or None.""" + for g in check.guards: + if isinstance(g, ElementGuard): + return g + return None + + +def _checks_for( + model_cls: type[BaseModel], +) -> tuple[list[Check], list[ModelCheck]]: + return build_checks(spec_for_model(model_cls)) + + +def _condition_of(check: ModelCheck) -> object: + """Return the condition of a RequireIf or ForbidIf descriptor.""" + desc = check.descriptor + assert isinstance(desc, (RequireIf, ForbidIf)), ( + f"Expected RequireIf or ForbidIf, got {type(desc).__name__}" + ) + return desc.condition + + +def _filter_nodes( + nodes: list[ModelCheck], + function: str | tuple[str, ...], + field_names: tuple[str, ...] | None = None, +) -> list[ModelCheck]: + functions = (function,) if isinstance(function, str) else function + return [ + n + for n in nodes + if model_constraint_function(n.descriptor) in functions + and (field_names is None or n.descriptor.field_names == field_names) + ] + + +def _union_checks( + name: str, union_type: object +) -> tuple[list[Check], list[ModelCheck]]: + return build_checks(union_spec_for(name, union_type)) + + +def _union_model_nodes(name: str, union_type: object) -> list[ModelCheck]: + _, model_nodes = _union_checks(name, union_type) + return model_nodes + + +class TestScalarChecks: + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(LiteralSubtypeModel) + return nodes + + def test_literal_produces_enum_check(self, nodes: list[Check]) -> None: + enum_nodes = [n for n in nodes if n.target == _path("subtype")] + assert len(enum_nodes) == 1 + node = enum_nodes[0] + descriptors = node.descriptors + funcs = [d.function for d in descriptors] + assert "check_required" in funcs + assert "check_enum" in funcs + + def test_optional_field_no_required_check(self, nodes: list[Check]) -> None: + name_nodes = [n for n in nodes if n.target == _path("name")] + for node in name_nodes: + funcs = [d.function for d in node.descriptors] + assert "check_required" not in funcs + + def test_required_comes_first_in_coalesce(self, nodes: list[Check]) -> None: + enum_nodes = [n for n in nodes if n.target == _path("subtype")] + node = enum_nodes[0] + funcs = [d.function for d in node.descriptors] + req_idx = funcs.index("check_required") + enum_idx = funcs.index("check_enum") + assert req_idx < enum_idx + + def test_enum_args_contain_literal_values(self, nodes: list[Check]) -> None: + enum_nodes = [n for n in nodes if n.target == _path("subtype")] + node = enum_nodes[0] + enum_desc = next(d for d in node.descriptors if d.function == "check_enum") + assert enum_desc.args == (("a", "b", "c"),) + + def test_optional_str_field_no_checks(self, nodes: list[Check]) -> None: + # name: str | None = None has no constraints, so no check node + name_nodes = [n for n in nodes if n.target == _path("name")] + assert len(name_nodes) == 0 + + +class _RequiredNewtypeModel(BaseModel): + country: CountryCodeAlpha2 + + +class TestRequiredNewtypeChecks: + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(_RequiredNewtypeModel) + return nodes + + def test_required_newtype_includes_check_required(self, nodes: list[Check]) -> None: + country_nodes = [n for n in nodes if n.target == _path("country")] + assert len(country_nodes) == 1 + funcs = [d.function for d in country_nodes[0].descriptors] + assert "check_required" in funcs + + def test_required_newtype_includes_newtype_function( + self, nodes: list[Check] + ) -> None: + country_nodes = [n for n in nodes if n.target == _path("country")] + funcs = [d.function for d in country_nodes[0].descriptors] + assert "check_pattern" in funcs + + def test_required_precedes_newtype_function(self, nodes: list[Check]) -> None: + country_nodes = [n for n in nodes if n.target == _path("country")] + funcs = [d.function for d in country_nodes[0].descriptors] + assert funcs.index("check_required") < funcs.index("check_pattern") + + +class _Color(str, Enum): + RED = "red" + GREEN = "green" + BLUE = "blue" + + +class EnumFieldModel(BaseModel): + color: _Color + + +class TestEnumKindChecks: + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(EnumFieldModel) + return nodes + + def test_enum_field_produces_check_enum(self, nodes: list[Check]) -> None: + enum_descs = [ + d for n in nodes for d in n.descriptors if d.function == "check_enum" + ] + assert len(enum_descs) == 1 + + def test_enum_field_uses_member_values(self, nodes: list[Check]) -> None: + enum_descs = [ + d for n in nodes for d in n.descriptors if d.function == "check_enum" + ] + assert enum_descs[0].args == (("red", "green", "blue"),) + + +class InnerModel(BaseModel): + value: str + count: int = Field(ge=0) + + +class OuterModel(BaseModel): + inner: InnerModel | None = None + + +class _ArrayElement(BaseModel): + tag: str + + +class _NullableWithArray(BaseModel): + items: list[_ArrayElement] | None = None + + +class _NullableArrayGrandparent(BaseModel): + parent: _NullableWithArray | None = None + + +class TestNullableParentGating: + """Required fields within optional struct parents get gated check_required.""" + + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(OuterModel) + return nodes + + def test_required_field_has_gated_check_required(self, nodes: list[Check]) -> None: + value_nodes = [n for n in nodes if n.target == _path("inner.value")] + req_descs = [ + d + for n in value_nodes + for d in n.descriptors + if d.function == "check_required" + ] + assert len(req_descs) == 1 + assert req_descs[0].gate == _path("inner") + + def test_non_check_required_descriptors_have_no_gate( + self, nodes: list[Check] + ) -> None: + count_nodes = [n for n in nodes if n.target == _path("inner.count")] + for node in count_nodes: + for desc in node.descriptors: + if desc.function != "check_required": + assert desc.gate is None + + def test_other_checks_still_present(self, nodes: list[Check]) -> None: + count_nodes = [n for n in nodes if n.target == _path("inner.count")] + assert len(count_nodes) >= 1 + funcs = [d.function for d in count_nodes[0].descriptors] + assert "check_bounds" in funcs + + +class TestArrayBoundaryResetsNullable: + """nullable_gate resets at array boundaries.""" + + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(_NullableArrayGrandparent) + return nodes + + def test_required_field_in_array_element_has_check_required( + self, nodes: list[Check] + ) -> None: + tag_nodes = [n for n in nodes if n.target == _path("parent.items[].tag")] + assert len(tag_nodes) >= 1 + funcs = [d.function for n in tag_nodes for d in n.descriptors] + assert "check_required" in funcs + + def test_array_element_required_has_no_gate(self, nodes: list[Check]) -> None: + tag_nodes = [n for n in nodes if n.target == _path("parent.items[].tag")] + req_descs = [ + d + for n in tag_nodes + for d in n.descriptors + if d.function == "check_required" + ] + assert len(req_descs) == 1 + assert req_descs[0].gate is None + + +class _OptionalNested(BaseModel): + mode: str + + +class _ElementWithOptional(BaseModel): + nested: _OptionalNested | None = None + + +class _ArrayWithOptionalNested(BaseModel): + items: list[_ElementWithOptional] + + +class TestArrayElementConditionalGate: + """Optional structs within array elements get gated check_required.""" + + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(_ArrayWithOptionalNested) + return nodes + + def test_required_in_optional_element_struct_has_gate( + self, nodes: list[Check] + ) -> None: + mode_nodes = [n for n in nodes if n.target == _path("items[].nested.mode")] + req_descs = [ + d + for n in mode_nodes + for d in n.descriptors + if d.function == "check_required" + ] + assert len(req_descs) == 1 + assert req_descs[0].gate == _path("items[].nested") + + +class TestStructRecursion: + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(OuterModel) + return nodes + + def test_recurses_into_model_fields(self, nodes: list[Check]) -> None: + paths = {n.target for n in nodes} + assert _path("inner.count") in paths + + def test_nested_field_uses_dot_path(self, nodes: list[Check]) -> None: + count_nodes = [n for n in nodes if n.target == _path("inner.count")] + assert len(count_nodes) == 1 + + +class ItemModel(BaseModel): + value: str + + +class ArrayModel(BaseModel): + items: Annotated[list[ItemModel], MinLen(1)] + + +class TestArrayChecks: + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(ArrayModel) + return nodes + + def test_array_min_length_is_scalar_shape(self, nodes: list[Check]) -> None: + length_nodes = [ + n + for n in nodes + if any(d.function == "check_array_min_length" for d in n.descriptors) + ] + assert len(length_nodes) == 1 + assert isinstance(length_nodes[0].target, ScalarPath) + + def test_array_element_field_uses_bracket_notation( + self, nodes: list[Check] + ) -> None: + paths = {n.target for n in nodes} + assert any(isinstance(p, ArrayPath) for p in paths) + + def test_array_element_subfield_path(self, nodes: list[Check]) -> None: + # ItemModel.value is required, so a check node for items[].value must exist + paths = {n.target for n in nodes} + assert _path("items[].value") in paths + + def test_array_level_check_has_no_inner_levels(self, nodes: list[Check]) -> None: + length_nodes = [ + n + for n in nodes + if any(d.function == "check_array_min_length" for d in n.descriptors) + ] + assert length_nodes[0].target == _path("items") + + def test_required_array_field_has_required_check(self, nodes: list[Check]) -> None: + # check_required on an array field is a column-level null check; its + # target is the scalar `items` column, not an element path. + required_nodes = [ + n + for n in nodes + if n.target == _path("items") + and any(d.function == "check_required" for d in n.descriptors) + ] + assert len(required_nodes) == 1 + + def test_array_element_subfield_has_single_check(self, nodes: list[Check]) -> None: + value_nodes = [n for n in nodes if n.target == _path("items[].value")] + assert len(value_nodes) == 1 + + +class _StringListModel(BaseModel): + tags: Annotated[list[str], MinLen(1)] + + +class _NestedListModel(BaseModel): + """list[list[ItemModel]] — both layers contribute MinLen + UniqueItems.""" + + items: Annotated[ + list[Annotated[list[InnerModel], MinLen(1), UniqueItemsConstraint()]], + MinLen(1), + UniqueItemsConstraint(), + ] + + +class _StringInListModel(BaseModel): + """list[Annotated[str, MinLen]] with outer list MinLen — inner is string MinLen.""" + + tags: Annotated[list[Annotated[str, MinLen(1)]], MinLen(1)] + + +_HierarchyItemList = NewType( + "_HierarchyItemList", + Annotated[list[InnerModel], MinLen(1), UniqueItemsConstraint()], +) + + +class _HierarchyLikeModel(BaseModel): + """Mirror of Division.hierarchies: inner list lives inside a NewType.""" + + hierarchies: Annotated[ + list[_HierarchyItemList], + MinLen(1), + UniqueItemsConstraint(), + ] + + +class TestListFieldNameSplitting: + """Column-level and element-level checks for list fields get distinct field names.""" + + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(_StringListModel) + return nodes + + def test_unique_labels_for_different_shapes(self, nodes: list[Check]) -> None: + labels = [(n.target, column_level_suffix(n)) for n in nodes] + assert len(labels) == len(set(labels)), f"Duplicate labels: {labels}" + + def test_min_length_check_carries_min_length_suffix( + self, nodes: list[Check] + ) -> None: + min_len_nodes = [ + n + for n in nodes + if any(d.function == "check_array_min_length" for d in n.descriptors) + ] + assert len(min_len_nodes) == 1 + assert min_len_nodes[0].target == _path("tags") + assert column_level_suffix(min_len_nodes[0]) == "_min_length" + + +def _node_for(nodes: list[Check], field: str, function: str) -> Check: + field_path = _path(field) + matching = [ + n + for n in nodes + if n.target == field_path and any(d.function == function for d in n.descriptors) + ] + assert len(matching) == 1, ( + f"expected exactly one node for field={field!r} function={function!r}, " + f"got {len(matching)}" + ) + return matching[0] + + +@pytest.mark.parametrize( + ("model_cls", "field"), + [ + (_NestedListModel, "items"), + (_HierarchyLikeModel, "hierarchies"), + ], + ids=["nested_list", "hierarchy_newtype"], +) +class TestPerLevelListConstraints: + """Each layer of `list[list[X]]` emits its own column-level check. + + Covers both raw nested lists (`_NestedListModel`) and the + NewType-wrapped variant (`_HierarchyLikeModel`, mirroring + `Division.hierarchies`). + """ + + def test_no_duplicate_labels(self, model_cls: type[BaseModel], field: str) -> None: + nodes, _ = _checks_for(model_cls) + labels = [(n.target, column_level_suffix(n)) for n in nodes] + assert len(labels) == len(set(labels)), f"Duplicate labels: {labels}" + + def test_outer_min_length_check( + self, model_cls: type[BaseModel], field: str + ) -> None: + nodes, _ = _checks_for(model_cls) + outer = _node_for(nodes, field, "check_array_min_length") + assert outer.target == _path(field) + + def test_inner_min_length_check( + self, model_cls: type[BaseModel], field: str + ) -> None: + nodes, _ = _checks_for(model_cls) + inner = _node_for(nodes, f"{field}[]", "check_array_min_length") + assert inner.target == _path(f"{field}[]") + + def test_outer_unique_check(self, model_cls: type[BaseModel], field: str) -> None: + nodes, _ = _checks_for(model_cls) + outer = _node_for(nodes, field, "check_struct_unique") + assert outer.target == _path(field) + + def test_inner_unique_check(self, model_cls: type[BaseModel], field: str) -> None: + nodes, _ = _checks_for(model_cls) + inner = _node_for(nodes, f"{field}[]", "check_struct_unique") + assert inner.target == _path(f"{field}[]") + + +class TestPerLevelScalarMinLen: + """list[Annotated[str, MinLen]] with outer list MinLen splits cleanly.""" + + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(_StringInListModel) + return nodes + + def test_outer_array_min_length(self, nodes: list[Check]) -> None: + outer = _node_for(nodes, "tags", "check_array_min_length") + assert outer.target == _path("tags") + + def test_inner_string_min_length(self, nodes: list[Check]) -> None: + inner = _node_for(nodes, "tags[]", "check_string_min_length") + assert inner.target == _path("tags[]") + + +class TestDescriptorDedupKey: + """Descriptor equality drives layer-level dedup via `dict.fromkeys`.""" + + def test_identical_descriptors_collapse(self) -> None: + desc = ExpressionDescriptor(function="check_array_min_length", args=(1,)) + assert list(dict.fromkeys([desc, desc])) == [desc] + + def test_distinct_descriptors_preserve_order(self) -> None: + first = ExpressionDescriptor(function="check_array_min_length", args=(1,)) + second = ExpressionDescriptor(function="check_struct_unique") + assert list(dict.fromkeys([first, second, first])) == [first, second] + + def test_different_args_are_distinct(self) -> None: + one = ExpressionDescriptor(function="check_array_min_length", args=(1,)) + two = ExpressionDescriptor(function="check_array_min_length", args=(2,)) + assert list(dict.fromkeys([one, two])) == [one, two] + + def test_different_gates_are_distinct(self) -> None: + ungated = ExpressionDescriptor(function="check_required") + gated = ExpressionDescriptor(function="check_required", gate=_path("parent")) + assert list(dict.fromkeys([ungated, gated])) == [ungated, gated] + + +class TestListOfNewtypeConstraintDispatch: + """Element-level MinLen from NewType inside a list dispatches as string check.""" + + @pytest.fixture + def nodes(self) -> list[Check]: + MyId = NewType("MyId", Annotated[str, MinLen(1)]) + + class ListOfIdModel(BaseModel): + ids: list[MyId] + + nodes, _ = _checks_for(ListOfIdModel) + return nodes + + def test_element_min_length_dispatches_as_string_check( + self, nodes: list[Check] + ) -> None: + """MinLen from the element NewType should produce check_string_min_length, not check_array_min_length.""" + all_funcs = [d.function for n in nodes for d in n.descriptors] + assert "check_string_min_length" in all_funcs + # check_array_min_length should NOT appear — there's no list-level MinLen + assert "check_array_min_length" not in all_funcs + + +class _InternalListNewtypeModel(BaseModel): + """Model with a NewType that wraps list[float] (list is inside the NewType).""" + + between: list[CountryCodeAlpha2] | None = None # outer list wrapping + + +class TestNewtypeWithInternalList: + """When a NewType IS a list, the check function handles the whole array.""" + + @pytest.fixture + def nodes(self) -> list[Check]: + class InternalListModel(BaseModel): + between: LinearlyReferencedRange | None = None + + nodes, _ = _checks_for(InternalListModel) + return nodes + + def test_internal_list_newtype_has_single_check(self, nodes: list[Check]) -> None: + between_nodes = [n for n in nodes if n.target == _path("between")] + assert len(between_nodes) == 1 + + def test_internal_list_newtype_has_three_descriptors( + self, nodes: list[Check] + ) -> None: + between_nodes = [n for n in nodes if n.target == _path("between")] + fns = [d.function for d in between_nodes[0].descriptors] + assert "check_linear_range_length" in fns + assert "check_linear_range_bounds" in fns + assert "check_linear_range_order" in fns + + +class TestBaseTypeDispatchInCheckBuilder: + """Base type dispatch generates element-level checks for HttpUrl/EmailStr.""" + + @pytest.fixture + def nodes(self) -> list[Check]: + class HttpUrlListModel(BaseModel): + websites: list[HttpUrl] | None = None + + nodes, _ = _checks_for(HttpUrlListModel) + return nodes + + def test_http_url_produces_check_url_format(self, nodes: list[Check]) -> None: + url_nodes = [ + n + for n in nodes + if any(d.function == "check_url_format" for d in n.descriptors) + ] + assert len(url_nodes) == 1 + + def test_http_url_element_check_is_array_shape(self, nodes: list[Check]) -> None: + url_nodes = [ + n + for n in nodes + if any(d.function == "check_url_format" for d in n.descriptors) + ] + assert isinstance(url_nodes[0].target, ArrayPath) + + +class _DeepInner(BaseModel): + field: str + + +class _ArrayElementWithNestedStruct(BaseModel): + nested: _DeepInner + + +class _DeepNestedArrayModel(BaseModel): + items: list[_ArrayElementWithNestedStruct] + + +class TestArrayElementNestedStructChecks: + """Struct fields inside array elements produce array-shaped checks.""" + + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(_DeepNestedArrayModel) + return nodes + + def test_nested_struct_field_path(self, nodes: list[Check]) -> None: + paths = {n.target for n in nodes} + assert _path("items[].nested.field") in paths + + +class _ArrayElementWithList(BaseModel): + tags: list[CountryCodeAlpha2] + + +class _ListInArrayModel(BaseModel): + items: list[_ArrayElementWithList] + + +class TestArrayElementListChecks: + """List fields inside array elements need nested iteration.""" + + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(_ListInArrayModel) + return nodes + + def test_list_subfield_element_checks_have_inner_levels( + self, nodes: list[Check] + ) -> None: + # Element-level check on a list field inside an outer array: target + # encodes both iterations explicitly as `items[].tags[]`. + element_nodes = [n for n in nodes if n.target == _path("items[].tags[]")] + assert len(element_nodes) >= 1 + + def test_list_subfield_column_path_is_enclosing_array( + self, nodes: list[Check] + ) -> None: + tag_nodes = [n for n in nodes if str(n.target).startswith("items[].tags")] + for node in tag_nodes: + assert isinstance(node.target, ArrayPath) + # the outermost iterated column is `items`, not the inner `tags` list + assert node.target.array_chunks[0] == ((), "items", 1) + + +class _ArrayElementWithNewtype(BaseModel): + country: CountryCodeAlpha2 + + +class _NewtypeInArrayModel(BaseModel): + items: list[_ArrayElementWithNewtype] + + +class TestArrayElementNewtypeChecks: + """Newtype fields inside array elements: shape=ARRAY, no inner_levels.""" + + @pytest.fixture + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(_NewtypeInArrayModel) + return nodes + + def test_newtype_subfield_has_single_check(self, nodes: list[Check]) -> None: + country_nodes = [n for n in nodes if n.target == _path("items[].country")] + assert len(country_nodes) == 1 + + +class TestModelLevelConstraints: + @pytest.fixture + def radio_model_nodes(self) -> list[ModelCheck]: + _, model_nodes = _checks_for(RadioModel) + return model_nodes + + @pytest.fixture + def require_any_model_nodes(self) -> list[ModelCheck]: + _, model_nodes = _checks_for(RequireAnyModel) + return model_nodes + + def test_radio_group_produces_model_check( + self, radio_model_nodes: list[ModelCheck] + ) -> None: + assert len(_filter_nodes(radio_model_nodes, "check_radio_group")) == 1 + + def test_radio_group_field_names(self, radio_model_nodes: list[ModelCheck]) -> None: + radio = _filter_nodes(radio_model_nodes, "check_radio_group")[0] + assert set(radio.descriptor.field_names) == {"a", "b"} + + def test_require_any_of_produces_model_check( + self, require_any_model_nodes: list[ModelCheck] + ) -> None: + assert len(_filter_nodes(require_any_model_nodes, "check_require_any_of")) == 1 + + def test_require_any_of_field_names( + self, require_any_model_nodes: list[ModelCheck] + ) -> None: + node = _filter_nodes(require_any_model_nodes, "check_require_any_of")[0] + assert set(node.descriptor.field_names) == {"x", "y"} + + def test_no_constraints_returns_empty_model_nodes(self) -> None: + _, model_nodes = _checks_for(LiteralSubtypeModel) + assert model_nodes == [] + + +class _SpeedStruct(BaseModel): + value: int + unit: str + + +@require_any_of("fast", "slow") +class _RequireAnyOfStructFields(BaseModel): + fast: _SpeedStruct | None = None + slow: _SpeedStruct | None = None + + +class TestRequireAnyOfStructUnwrapping: + """require_any_of on struct fields must reference the leaf scalar, not the struct.""" + + @pytest.fixture + def node(self) -> ModelCheck: + _, model_nodes = _checks_for(_RequireAnyOfStructFields) + nodes = _filter_nodes(model_nodes, "check_require_any_of") + assert len(nodes) == 1 + return nodes[0] + + def test_field_names_use_leaf_path(self, node: ModelCheck) -> None: + assert set(node.descriptor.field_names) == {"fast.value", "slow.value"} + + +class _SyntheticUnionFixtures: + """Discriminated-union models exercising union check generation.""" + + class Base(BaseModel): + kind: str + + class TypeA(Base): + kind: Literal["a"] = "a" + a_field: Literal["x", "y"] | None = None + + class TypeB(Base): + kind: Literal["b"] = "b" + b_field: Literal["p", "q"] | None = None + + SyntheticUnion = Annotated[ + Union[TypeA, TypeB], # noqa: UP007 + FieldInfo(discriminator="kind"), + ] + + @require_any_of("p", "q") + class ConstrainedMember(Base): + kind: Literal["c"] = "c" + p: str | None = None + q: str | None = None + + ConstrainedUnion = Annotated[ + Union[TypeA, ConstrainedMember], # noqa: UP007 + FieldInfo(discriminator="kind"), + ] + + class MemberX(Base): + kind: Literal["x"] = "x" + shared_name: Literal["x1", "x2"] + + class MemberY(Base): + kind: Literal["y"] = "y" + shared_name: Literal["y1", "y2"] + + class MemberZ(Base): + kind: Literal["z"] = "z" + + ThreeWayUnion = Annotated[ + Union[MemberX, MemberY, MemberZ], # noqa: UP007 + FieldInfo(discriminator="kind"), + ] + + class MixedRequired(Base): + kind: Literal["r"] = "r" + mixed_field: str + + class MixedOptional(Base): + kind: Literal["o"] = "o" + mixed_field: str | None = None + + class MixedAbsent(Base): + kind: Literal["a"] = "a" + + MixedRequirednessUnion = Annotated[ + Union[MixedRequired, MixedOptional, MixedAbsent], # noqa: UP007 + FieldInfo(discriminator="kind"), + ] + + class AllVarA(Base): + kind: Literal["a"] = "a" + everywhere: str | None = None + + class AllVarB(Base): + kind: Literal["b"] = "b" + everywhere: str | None = None + + AllVariantsUnion = Annotated[ + Union[AllVarA, AllVarB], # noqa: UP007 + FieldInfo(discriminator="kind"), + ] + + @require_any_of("fast", "slow") + @forbid_if(["restrictions"], FieldEqCondition("gated", True)) + class MemberWithModelConstraints(Base): + """Union member carrying model constraints over struct/compound fields.""" + + kind: Literal["m"] = "m" + gated: bool = False + fast: _SpeedStruct | None = None + slow: _SpeedStruct | None = None + restrictions: list[str] | None = None + + MemberConstraintUnion = Annotated[ + Union[TypeA, MemberWithModelConstraints], # noqa: UP007 + FieldInfo(discriminator="kind"), + ] + + class PlainMember(Base): + kind: Literal["p"] = "p" + + +class TestSyntheticUnionChecks: + @pytest.fixture + def field_nodes(self) -> list[Check]: + nodes, _ = _union_checks("Synthetic", _SyntheticUnionFixtures.SyntheticUnion) + return nodes + + def test_variant_field_gets_variant_values(self, field_nodes: list[Check]) -> None: + a_nodes = [n for n in field_nodes if n.target == _path("a_field")] + assert len(a_nodes) > 0 + for node in a_nodes: + assert node.guards == (ColumnGuard(discriminator="kind", values=("a",)),) + + def test_shared_field_has_no_variant_values(self, field_nodes: list[Check]) -> None: + kind_nodes = [n for n in field_nodes if n.target == _path("kind")] + for node in kind_nodes: + assert node.guards == () + + def test_b_field_gets_b_variant_value(self, field_nodes: list[Check]) -> None: + b_nodes = [n for n in field_nodes if n.target == _path("b_field")] + assert len(b_nodes) > 0 + for node in b_nodes: + assert node.guards == (ColumnGuard(discriminator="kind", values=("b",)),) + + def test_variant_nodes_carry_discriminator_field( + self, field_nodes: list[Check] + ) -> None: + variant_nodes = [n for n in field_nodes if n.guards] + for node in variant_nodes: + for guard in node.guards: + assert guard.discriminator == "kind" + + @pytest.fixture + def model_nodes(self) -> list[ModelCheck]: + return _union_model_nodes("Synthetic", _SyntheticUnionFixtures.SyntheticUnion) + + @pytest.mark.parametrize( + ("field_name", "expected_value"), + [("a_field", "b"), ("b_field", "a")], + ) + def test_variant_field_gets_forbid_if( + self, + model_nodes: list[ModelCheck], + field_name: str, + expected_value: str, + ) -> None: + forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if", (field_name,)) + assert len(forbid_nodes) == 1 + condition = _condition_of(forbid_nodes[0]) + assert isinstance(condition, FieldEqCondition) + assert condition.field_name == "kind" + assert condition.value == expected_value + + def test_forbid_if_nodes_are_top_level(self, model_nodes: list[ModelCheck]) -> None: + forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if") + assert len(forbid_nodes) == 2 + for node in forbid_nodes: + assert node.target == ScalarPath() + + +class TestUnionMemberModelConstraints: + @pytest.fixture + def model_nodes(self) -> list[ModelCheck]: + return _union_model_nodes( + "Constrained", _SyntheticUnionFixtures.ConstrainedUnion + ) + + def test_member_model_constraints_collected( + self, model_nodes: list[ModelCheck] + ) -> None: + assert len(_filter_nodes(model_nodes, "check_require_any_of")) == 1 + + def test_member_constraint_tagged_with_arm( + self, model_nodes: list[ModelCheck] + ) -> None: + """Constraint from ConstrainedMember carries that member's discriminator value.""" + require_any_of_nodes = _filter_nodes(model_nodes, "check_require_any_of") + assert len(require_any_of_nodes) == 1 + assert require_any_of_nodes[0].arm == "c" + + def test_exclusivity_checks_have_no_arm( + self, model_nodes: list[ModelCheck] + ) -> None: + """Synthesized forbid_if/require_if checks apply to every arm.""" + exclusivity_nodes = _filter_nodes( + model_nodes, ("check_forbid_if", "check_require_if") + ) + assert exclusivity_nodes + for node in exclusivity_nodes: + assert node.arm is None + + +class TestUnionMemberStructAndCompoundConstraints: + """Member-level constraints on struct/compound fields dispatch with real shapes. + + `@require_any_of` over struct fields must unwrap to the first required + leaf scalar; `@forbid_if` over a compound field must populate + `field_shapes`. Both depend on the member being run through real + extraction rather than stubbed proxies -- a latent gap, since no real + schema member currently carries a model-level constraint decorator. + """ + + @pytest.fixture + def model_nodes(self) -> list[ModelCheck]: + return _union_model_nodes( + "MemberConstraint", _SyntheticUnionFixtures.MemberConstraintUnion + ) + + def test_require_any_of_unwraps_struct_leaf( + self, model_nodes: list[ModelCheck] + ) -> None: + nodes = _filter_nodes(model_nodes, "check_require_any_of") + assert len(nodes) == 1 + assert set(nodes[0].descriptor.field_names) == {"fast.value", "slow.value"} + + def test_forbid_if_populates_compound_field_shapes( + self, model_nodes: list[ModelCheck] + ) -> None: + # Exclusivity logic also emits a forbid_if for `restrictions`, but + # gated on the discriminator; the member-level constraint is the + # one whose condition references `gated`. + forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if", ("restrictions",)) + member_level = [ + n + for n in forbid_nodes + if isinstance((cond := _condition_of(n)), FieldEqCondition) + and cond.field_name == "gated" + ] + assert len(member_level) == 1 + descriptor = member_level[0].descriptor + assert isinstance(descriptor, ForbidIf) + assert "restrictions" in dict(descriptor.field_shapes) + + +@require_any_of("max_speed", "min_speed") +class _SpeedLimitElement(BaseModel): + """Element model with its own @require_any_of constraint.""" + + max_speed: int | None = None + min_speed: int | None = None + + +class _VariantWithConstrainedList(_SyntheticUnionFixtures.Base): + """Union member with a variant-specific list of constrained sub-models.""" + + kind: Literal["v"] = "v" + speed_limits: list[_SpeedLimitElement] | None = None + + +_VariantFieldConstraintUnion = Annotated[ + Union[_VariantWithConstrainedList, _SyntheticUnionFixtures.PlainMember], # noqa: UP007 + FieldInfo(discriminator="kind"), +] + + +class TestVariantSpecificFieldDiscoveredModelConstraints: + """Model constraints discovered through a variant-specific field carry the contributing arm. + + A `@require_any_of` declared on an element model of a list field that + appears only in one union arm must be tagged with that arm, not + propagated to every arm. + """ + + @pytest.fixture + def model_nodes(self) -> list[ModelCheck]: + return _union_model_nodes( + "VariantFieldConstraint", _VariantFieldConstraintUnion + ) + + def test_field_discovered_constraint_tagged_with_arm( + self, model_nodes: list[ModelCheck] + ) -> None: + nodes = [ + n + for n in _filter_nodes(model_nodes, "check_require_any_of") + if n.target == _path("speed_limits[]") + ] + assert len(nodes) == 1 + assert nodes[0].arm == "v" + + +class _VariantWithConstrainedModelRef(_SyntheticUnionFixtures.Base): + """Variant-specific direct (non-list) model ref with model-level constraint.""" + + kind: Literal["d"] = "d" + speed: _SpeedLimitElement | None = None + + +_DirectModelRefConstraintUnion = Annotated[ + Union[_VariantWithConstrainedModelRef, _SyntheticUnionFixtures.PlainMember], # noqa: UP007 + FieldInfo(discriminator="kind"), +] + + +class TestVariantSpecificDirectModelRefConstraint: + """Variant-specific non-list `ModelRef` with a constrained sub-model is unsupported. + + The direct-ref path routes through `_recurse_into_model` rather + than the array branch of `_walk_field_shape`, and pure struct + nesting can't anchor a real model constraint -- the dispatch + raises `NotImplementedError`. Distinct from the `list[Model]` + case in `TestVariantSpecificFieldDiscoveredModelConstraints`, + which is supported. + """ + + def test_direct_modelref_constraint_raises(self) -> None: + # Pure struct nesting can't anchor a real model constraint; today + # the only constraint kind that survives struct nesting raises. + with pytest.raises( + NotImplementedError, match="Model constraint on struct-nested" + ): + _union_model_nodes( + "DirectModelRefConstraint", _DirectModelRefConstraintUnion + ) + + +class _OuterWithStructNestedUnion(BaseModel): + """Non-list `UnionRef` field reaches a union with a constrained member.""" + + nested: _SyntheticUnionFixtures.ConstrainedUnion + + +class TestStructNestedUnionWithConstraint: + """Non-list `UnionRef` reaching a union with model checks is unsupported. + + `_recurse_into_union` mirrors `_recurse_into_model`'s guard: when + the prefix is struct-nested (no `ArrayPath` segment) and the union + would emit either union-level constraints or synthesized + exclusivity checks (`check_forbid_if`/`check_require_if`), the + dispatch raises because `_model_constraint_target` would collapse + the anchor to the row root with field names that don't exist + there. This fixture exercises the union-level branch; the + exclusivity branch isn't covered by a synthetic fixture today + because the dual-trigger raise body is one statement. + """ + + def test_struct_nested_union_constraint_raises(self) -> None: + with pytest.raises( + NotImplementedError, match="Model constraint on struct-nested" + ): + build_checks(spec_for_model(_OuterWithStructNestedUnion)) + + +class TestStructNestedUnionWithVariantFields: + """Struct-nested union producing gated field checks is unsupported. + + A `ColumnGuard` carries a bare discriminator name that renders as + `F.col("")` -- a top-level column access that is wrong + when the union is reached through a plain struct field. Raising loudly + is safer than emitting a mis-gated check. + + Distinct from `TestStructNestedUnionWithConstraint`: that class covers + model/exclusivity checks; this class covers variant-gated field checks + (the silent-failure path the previous guard missed). + + The trigger spec is built manually (not via `spec_for_model`) because + Pydantic strips the `Annotated[Union[...], FieldInfo(discriminator=...)]` + wrapper from `model_fields`, causing the inline extraction path to lose + `discriminator_mapping`. Constructing `UnionRef(union=...)` directly + with a fully-extracted union spec (via `union_spec_for`) replicates the + state that a future extraction path that preserves discriminator metadata + would produce. + """ + + @pytest.fixture(scope="class") + def discriminated_union_ref_spec(self) -> RecordSpec: + """A `RecordSpec` whose `nested` field holds a `UnionRef` with a full discriminator.""" + union_spec = union_spec_for("Synthetic", _SyntheticUnionFixtures.SyntheticUnion) + field = FieldSpec( + name="nested", + shape=UnionRef(union=union_spec), + description=None, + is_required=True, + is_optional=False, + ) + return RecordSpec(name="Outer", description=None, fields=[field]) + + def test_struct_nested_union_variant_fields_raises( + self, discriminated_union_ref_spec: RecordSpec + ) -> None: + with pytest.raises(NotImplementedError, match="ColumnGuard"): + build_checks(discriminated_union_ref_spec) + + def test_row_root_union_with_variant_fields_succeeds(self) -> None: + """Row-root union (empty `ScalarPath`) must still build checks without raising.""" + field_checks, _ = _union_checks( + "Synthetic", _SyntheticUnionFixtures.SyntheticUnion + ) + assert any(n.guards for n in field_checks) + + def test_array_reached_union_with_variant_fields_succeeds(self) -> None: + """Array-reached union (`ArrayPath` prefix) must still build checks without raising.""" + field_checks, _ = _checks_for(_ListUnionContainer) + assert any(n.guards for n in field_checks) + + +class _NestedInnerBase(BaseModel): + inner_kind: str + + +class _NestedInnerArmA(_NestedInnerBase): + inner_kind: Literal["i_a"] = "i_a" + a_only: str | None = None + + +@require_any_of("first", "second") +class _NestedInnerArmB(_NestedInnerBase): + """Inner-union arm with its own model-level constraint.""" + + inner_kind: Literal["i_b"] = "i_b" + first: str | None = None + second: str | None = None + + +_NestedInnerUnion = Annotated[ + Union[_NestedInnerArmA, _NestedInnerArmB], # noqa: UP007 + FieldInfo(discriminator="inner_kind"), +] + + +class _OuterArmWithInnerUnion(_SyntheticUnionFixtures.Base): + """Outer-union arm that wraps a nested union via a list field.""" + + kind: Literal["n"] = "n" + inners: list[_NestedInnerUnion] | None = None + + +_NestedUnionViaVariantField = Annotated[ + Union[_OuterArmWithInnerUnion, _SyntheticUnionFixtures.PlainMember], # noqa: UP007 + FieldInfo(discriminator="kind"), +] + + +class TestNestedUnionThroughVariantField: + """Inner-union member constraints inherit the outer-union arm. + + Reached through a variant-specific field carrying a nested union, + the inner member's `@require_any_of` must be tagged with the outer + arm ('n'), not the inner discriminator value ('i_b'). The outermost + union's discriminator is the only one per-arm test filtering keys + on. + """ + + @pytest.fixture + def model_nodes(self) -> list[ModelCheck]: + return _union_model_nodes( + "NestedUnionViaVariantField", _NestedUnionViaVariantField + ) + + def test_inner_member_constraint_tagged_with_outer_arm( + self, model_nodes: list[ModelCheck] + ) -> None: + require_any_of_nodes = _filter_nodes(model_nodes, "check_require_any_of") + assert len(require_any_of_nodes) == 1 + assert require_any_of_nodes[0].arm == "n" + + +class _MultiArmContributorA(_SyntheticUnionFixtures.Base): + kind: Literal["a"] = "a" + shared_limits: list[_SpeedLimitElement] | None = None + + +class _MultiArmContributorB(_SyntheticUnionFixtures.Base): + kind: Literal["b"] = "b" + shared_limits: list[_SpeedLimitElement] | None = None + + +class _MultiArmThirdMember(_SyntheticUnionFixtures.Base): + """Third arm that does NOT contribute the shared field.""" + + kind: Literal["c"] = "c" + + +_MultiArmVariantSourcesUnion = Annotated[ + Union[ # noqa: UP007 + _MultiArmContributorA, _MultiArmContributorB, _MultiArmThirdMember + ], + FieldInfo(discriminator="kind"), +] + + +class TestMultiArmVariantSourcesPolicy: + """Tombstone: a 2-of-N variant-specific field collapses to `arm=None`. + + No real schema today declares a variant-specific field on a proper + subset of arms (2-of-N). When/if that pattern surfaces with a + sub-model carrying its own model constraint, the current policy + routes the constraint to every arm rather than the intersection -- + including arms the field doesn't belong to. This pins the + behaviour explicitly so the gap surfaces if anyone treats it as + correct or relies on it. See `_singleton_arm` in `check_builder.py`. + """ + + @pytest.fixture + def model_nodes(self) -> list[ModelCheck]: + return _union_model_nodes( + "MultiArmVariantSources", _MultiArmVariantSourcesUnion + ) + + def test_multi_arm_field_discovered_constraint_has_no_arm( + self, model_nodes: list[ModelCheck] + ) -> None: + nodes = [ + n + for n in _filter_nodes(model_nodes, "check_require_any_of") + if n.target == _path("shared_limits[]") + ] + assert len(nodes) == 1 + # The 2-of-N case can't pick a single arm, so the constraint + # carries arm=None -- broadcasting to every arm, including the + # third member that doesn't declare shared_limits at all. + # Tracked for resolution if/when a real schema surfaces this. + assert nodes[0].arm is None + + +class TestGroupedExclusivityChecks: + """A required field with the same name in 2 of 3 variants (different types) groups correctly.""" + + @pytest.fixture + def model_nodes(self) -> list[ModelCheck]: + return _union_model_nodes("ThreeWay", _SyntheticUnionFixtures.ThreeWayUnion) + + def test_grouped_field_forbid_if_for_excluded_variant( + self, model_nodes: list[ModelCheck] + ) -> None: + forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if", ("shared_name",)) + assert len(forbid_nodes) == 1 + condition = _condition_of(forbid_nodes[0]) + assert isinstance(condition, FieldEqCondition) + assert condition.field_name == "kind" + assert condition.value == "z" + + def test_grouped_field_require_if_per_variant( + self, model_nodes: list[ModelCheck] + ) -> None: + require_nodes = _filter_nodes(model_nodes, "check_require_if", ("shared_name",)) + assert len(require_nodes) == 2 + conditions = set() + for node in require_nodes: + cond = _condition_of(node) + assert isinstance(cond, FieldEqCondition) + conditions.add(cond.value) + assert conditions == {"x", "y"} + + +class TestMixedRequirednessExclusivity: + """Same-named field required in one variant, optional in another.""" + + @pytest.fixture + def model_nodes(self) -> list[ModelCheck]: + return _union_model_nodes( + "Mixed", _SyntheticUnionFixtures.MixedRequirednessUnion + ) + + def test_require_if_only_for_required_variant( + self, model_nodes: list[ModelCheck] + ) -> None: + require_nodes = _filter_nodes(model_nodes, "check_require_if", ("mixed_field",)) + assert len(require_nodes) == 1 + condition = _condition_of(require_nodes[0]) + assert isinstance(condition, FieldEqCondition) + assert condition.value == "r" + + def test_forbid_if_for_absent_variant(self, model_nodes: list[ModelCheck]) -> None: + forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if", ("mixed_field",)) + assert len(forbid_nodes) == 1 + condition = _condition_of(forbid_nodes[0]) + assert isinstance(condition, FieldEqCondition) + assert condition.value == "a" + + +class TestExclusivityEdgeCases: + def test_no_discriminator_produces_zero_exclusivity_nodes(self) -> None: + """Union without discriminator_mapping produces no exclusivity checks.""" + spec = replace( + extract_union("Synthetic", _SyntheticUnionFixtures.SyntheticUnion), + discriminator_mapping=None, + discriminator_field=None, + ) + _, model_nodes = build_checks(spec) + forbid = _filter_nodes(model_nodes, "check_forbid_if") + require = _filter_nodes(model_nodes, "check_require_if") + assert len(forbid) + len(require) == 0 + + def test_field_in_all_variants_no_exclusivity(self) -> None: + """Field present in every variant via variant_sources produces no exclusivity checks.""" + _, model_nodes = _union_checks( + "AllVariants", _SyntheticUnionFixtures.AllVariantsUnion + ) + forbid = _filter_nodes(model_nodes, "check_forbid_if") + require = _filter_nodes(model_nodes, "check_require_if") + assert len(forbid) + len(require) == 0 + + +@require_any_of("x", "y") +class _ArrayElementWithConstraint(BaseModel): + x: str | None = None + y: str | None = None + + +class _ArrayOfConstrainedModel(BaseModel): + items: list[_ArrayElementWithConstraint] + + +class _OptionalArrayOfConstrainedModel(BaseModel): + items: list[_ArrayElementWithConstraint] | None = None + + +@require_any_of("a", "b") +class _NestedConstrainedStruct(BaseModel): + a: str | None = None + b: str | None = None + + +class _ArrayElementWithConstrainedNested(BaseModel): + nested: _NestedConstrainedStruct + + +class _ArrayOfNestedConstrained(BaseModel): + items: list[_ArrayElementWithConstrainedNested] + + +@require_any_of("a", "b") +class _InnerConstrainedElement(BaseModel): + a: str | None = None + b: str | None = None + + +class _OuterElementWithConstrainedList(BaseModel): + things: list[_InnerConstrainedElement] + + +class _DoubleNestedConstrained(BaseModel): + items: list[_OuterElementWithConstrainedList] + + +def _require_any_node_for(model_cls: type[BaseModel]) -> ModelCheck: + _, model_nodes = _checks_for(model_cls) + nodes = _filter_nodes(model_nodes, "check_require_any_of") + assert len(nodes) == 1 + return nodes[0] + + +@pytest.mark.parametrize( + ("model_cls", "expected_target"), + [ + pytest.param(_ArrayOfConstrainedModel, _path("items[]"), id="direct_element"), + pytest.param( + _ArrayOfNestedConstrained, _path("items[].nested"), id="nested_struct" + ), + ], +) +class TestArrayContextModelConstraints: + """Model constraints on array-element (or nested struct) models produce array-context ModelChecks.""" + + def test_produces_model_check_node( + self, model_cls: type[BaseModel], expected_target: FieldPath + ) -> None: + node = _require_any_node_for(model_cls) + assert model_constraint_function(node.descriptor) == "check_require_any_of" + + def test_target( + self, model_cls: type[BaseModel], expected_target: FieldPath + ) -> None: + node = _require_any_node_for(model_cls) + assert node.target == expected_target + + +class TestDoubleNestedArrayModelConstraints: + """Model constraints on list[] elements nested inside another array use nested geometry.""" + + def test_target_is_nested_inner_array(self) -> None: + # `things` is itself an ArraySegment, so the constraint's target + # iterates items[] then things[] with no struct nav between. + node = _require_any_node_for(_DoubleNestedConstrained) + assert node.target == _path("items[].things[]") + + +class TestSegmentUnionChecks: + @pytest.fixture(scope="class") + def segment_spec(self) -> ModelSpec: + return discover_feature("Segment") + + @pytest.fixture(scope="class") + def segment_checks( + self, segment_spec: ModelSpec + ) -> tuple[list[Check], list[ModelCheck]]: + return build_checks(segment_spec) + + @pytest.fixture(scope="class") + def field_nodes( + self, segment_checks: tuple[list[Check], list[ModelCheck]] + ) -> list[Check]: + return segment_checks[0] + + @pytest.fixture(scope="class") + def model_nodes( + self, segment_checks: tuple[list[Check], list[ModelCheck]] + ) -> list[ModelCheck]: + return segment_checks[1] + + def test_produces_variant_gated_checks(self, field_nodes: list[Check]) -> None: + variant_nodes = [n for n in field_nodes if n.guards] + assert len(variant_nodes) > 0 + + def test_shared_fields_have_no_variant_values( + self, field_nodes: list[Check] + ) -> None: + subtype_nodes = [n for n in field_nodes if n.target == _path("subtype")] + for node in subtype_nodes: + assert node.guards == () + + def test_speed_limits_require_any_of_in_model_nodes( + self, model_nodes: list[ModelCheck] + ) -> None: + speed_limit_nodes = [ + n + for n in _filter_nodes(model_nodes, "check_require_any_of") + if n.target == _path("speed_limits[]") + ] + assert len(speed_limit_nodes) >= 1 + + def test_destinations_require_any_of_in_model_nodes( + self, model_nodes: list[ModelCheck] + ) -> None: + dest_nodes = [ + n + for n in _filter_nodes(model_nodes, "check_require_any_of") + if n.target == _path("destinations[]") + ] + assert len(dest_nodes) >= 1 + + def test_speed_limits_when_require_any_of_in_model_nodes( + self, model_nodes: list[ModelCheck] + ) -> None: + when_nodes = [ + n + for n in _filter_nodes(model_nodes, "check_require_any_of") + if n.target == _path("speed_limits[].when") + ] + assert len(when_nodes) >= 1 + + @pytest.mark.parametrize( + ("field_name", "expected_subtype"), + [("road_surface", "road"), ("rail_flags", "rail")], + ) + def test_single_variant_field_forbid_if( + self, + model_nodes: list[ModelCheck], + field_name: str, + expected_subtype: str, + ) -> None: + forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if", (field_name,)) + assert len(forbid_nodes) == 1 + condition = _condition_of(forbid_nodes[0]) + assert isinstance(condition, Not) + assert isinstance(condition.inner, FieldEqCondition) + assert condition.inner.field_name == "subtype" + assert condition.inner.value == expected_subtype + + def test_class_forbid_if_for_water(self, model_nodes: list[ModelCheck]) -> None: + forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if", ("class",)) + assert len(forbid_nodes) == 1 + condition = _condition_of(forbid_nodes[0]) + assert isinstance(condition, FieldEqCondition) + assert condition.field_name == "subtype" + assert condition.value == "water" + + def test_class_require_if_for_road_and_rail( + self, model_nodes: list[ModelCheck] + ) -> None: + require_nodes = _filter_nodes(model_nodes, "check_require_if", ("class",)) + assert len(require_nodes) == 2 + conditions = [_condition_of(n) for n in require_nodes] + assert all(isinstance(c, FieldEqCondition) for c in conditions) + values = {c.value for c in conditions if isinstance(c, FieldEqCondition)} + assert values == {"road", "rail"} + + def test_nested_union_discriminator_preserved( + self, field_nodes: list[Check] + ) -> None: + """Inner VehicleSelector discriminator survives outer Segment annotation. + + Vehicle unit checks inside variant-specific fields (speed_limits, + prohibited_transitions) need both the outer subtype guard and the + inner dimension discriminator. The outer annotation must not + clobber the inner one. + """ + unit_nodes = [ + n for n in field_nodes if "vehicle[].unit" in str(n.target) and n.guards + ] + assert len(unit_nodes) > 0, "Expected variant-gated vehicle unit nodes" + + for node in unit_nodes: + inner = _element_guard(node) + assert inner is not None, ( + f"{node.target}: inner discriminator should be element-level, " + f"got guards {node.guards}" + ) + assert inner.discriminator == "dimension", ( + f"{node.target}: inner discriminator should be 'dimension', " + f"got {inner.discriminator!r}" + ) + + # Variant-specific fields (speed_limits, prohibited_transitions) + # also need the outer subtype guard. + speed_unit_nodes = [n for n in unit_nodes if "speed_limits" in str(n.target)] + for node in speed_unit_nodes: + outer = _column_guard(node) + assert outer is not None, f"{node.target}: missing outer subtype guard" + assert outer.discriminator == "subtype", ( + f"{node.target}: outer discriminator should be 'subtype', " + f"got {outer.discriminator!r}" + ) + + def test_segment_vehicle_selector_field_checks( + self, field_nodes: list[Check] + ) -> None: + """VehicleSelector fields appear with correct nesting.""" + vehicle_nodes = [n for n in field_nodes if "vehicle[]" in str(n.target)] + assert len(vehicle_nodes) > 0 + + dim_nodes = [n for n in vehicle_nodes if "dimension" in str(n.target)] + assert any("speed_limits" in str(n.target) for n in dim_nodes) + assert any("access_restrictions" in str(n.target) for n in dim_nodes) + + for node in dim_nodes: + assert isinstance(node.target, ArrayPath) + # vehicle[] is nested inside an outer array (speed_limits, etc.), + # so the struct nav to `dimension` lands in the target's leaf. + assert len(node.target.leaf) >= 1 + + def test_segment_vehicle_selector_exclusivity( + self, model_nodes: list[ModelCheck] + ) -> None: + """VehicleSelector produces forbid_if/require_if for unit field.""" + vehicle_forbid = [ + n + for n in _filter_nodes(model_nodes, "check_forbid_if") + if "unit" in n.descriptor.field_names and isinstance(n.target, ArrayPath) + ] + assert len(vehicle_forbid) > 0 + + vehicle_require = [ + n + for n in _filter_nodes(model_nodes, "check_require_if") + if "unit" in n.descriptor.field_names and isinstance(n.target, ArrayPath) + ] + assert len(vehicle_require) > 0 + + def test_segment_vehicle_selector_exclusivity_has_inner_levels( + self, model_nodes: list[ModelCheck] + ) -> None: + """VehicleSelector exclusivity checks use nested geometry to reach vehicle[].""" + vehicle_constraint_nodes = [ + n + for n in _filter_nodes(model_nodes, ("check_forbid_if", "check_require_if")) + if "unit" in n.descriptor.field_names and isinstance(n.target, ArrayPath) + ] + for node in vehicle_constraint_nodes: + assert isinstance(node.target, ArrayPath) + # The target reaches the inner vehicle[] via a second iteration: + # one inner level navigating `when` to the `vehicle` array. + iter_paths = node.target.iter_struct_paths + assert len(iter_paths) == 1 + assert "when" in iter_paths[0] + assert "vehicle" in iter_paths[0] + + +class _InnerBase(BaseModel): + kind: str + + +class _InnerA(_InnerBase): + kind: Literal["a"] = "a" + a_field: str + + +class _InnerB(_InnerBase): + kind: Literal["b"] = "b" + b_field: int = Field(ge=0) + + +_InnerUnion = Annotated[ + _InnerA | _InnerB, + Field(discriminator="kind"), +] + + +class _Wrapper(BaseModel): + items: list[_InnerUnion] + + +class TestUnionInsideArray: + """UNION-kind fields nested inside list[] produce variant-gated checks.""" + + @pytest.fixture(scope="class") + def results(self) -> tuple[list[Check], list[ModelCheck]]: + return build_checks(spec_for_model(_Wrapper)) + + @pytest.fixture(scope="class") + def field_nodes(self, results: tuple[list[Check], list[ModelCheck]]) -> list[Check]: + return results[0] + + @pytest.fixture(scope="class") + def model_nodes( + self, results: tuple[list[Check], list[ModelCheck]] + ) -> list[ModelCheck]: + return results[1] + + @pytest.fixture(scope="class") + def a_nodes(self, field_nodes: list[Check]) -> list[Check]: + return [n for n in field_nodes if n.target == _path("items[].a_field")] + + @pytest.fixture(scope="class") + def b_nodes(self, field_nodes: list[Check]) -> list[Check]: + return [n for n in field_nodes if n.target == _path("items[].b_field")] + + def test_a_field_check_produced(self, a_nodes: list[Check]) -> None: + assert len(a_nodes) >= 1 + + def test_a_field_is_array_shape(self, a_nodes: list[Check]) -> None: + assert isinstance(a_nodes[0].target, ArrayPath) + + def test_a_field_target_is_items(self, a_nodes: list[Check]) -> None: + assert a_nodes[0].target == _path("items[].a_field") + + def test_a_field_guard(self, a_nodes: list[Check]) -> None: + assert a_nodes[0].guards == (ElementGuard(discriminator="kind", values=("a",)),) + + def test_a_nodes_have_array_shape(self, a_nodes: list[Check]) -> None: + assert all(isinstance(n.target, ArrayPath) for n in a_nodes) + + def test_b_field_check_produced(self, b_nodes: list[Check]) -> None: + assert len(b_nodes) >= 1 + + def test_b_field_guard(self, b_nodes: list[Check]) -> None: + assert b_nodes[0].guards == (ElementGuard(discriminator="kind", values=("b",)),) + + def test_b_nodes_have_array_shape(self, b_nodes: list[Check]) -> None: + assert all(isinstance(n.target, ArrayPath) for n in b_nodes) + + def test_forbid_nodes_produced(self, model_nodes: list[ModelCheck]) -> None: + forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if") + assert len(forbid_nodes) > 0 + + def test_forbid_nodes_have_array_column_path( + self, model_nodes: list[ModelCheck] + ) -> None: + forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if") + for node in forbid_nodes: + assert node.target == _path("items[]") + + def test_require_if_model_nodes_have_array_column_path( + self, model_nodes: list[ModelCheck] + ) -> None: + require_nodes = _filter_nodes(model_nodes, "check_require_if") + for node in require_nodes: + assert node.target == _path("items[]") + + +class TestTopLevelUnionColumnPath: + """Top-level union (not inside array) exclusivity nodes have column_path=None.""" + + @pytest.fixture(scope="class") + def model_nodes(self) -> list[ModelCheck]: + return _union_model_nodes("Synthetic", _SyntheticUnionFixtures.SyntheticUnion) + + def test_forbid_if_column_path_is_none(self, model_nodes: list[ModelCheck]) -> None: + forbid_nodes = _filter_nodes(model_nodes, "check_forbid_if") + assert len(forbid_nodes) > 0 + for node in forbid_nodes: + assert node.target == ScalarPath() + + def test_require_if_column_path_is_none( + self, model_nodes: list[ModelCheck] + ) -> None: + require_nodes = _filter_nodes(model_nodes, "check_require_if") + for node in require_nodes: + assert node.target == ScalarPath() + + +class _ListUnionContainer(BaseModel): + """Top-level list of a discriminated union. + + The variant fields live inside each list element, so variant gating + must reference the element-level discriminator (`el["kind"]`), not a + top-level column (`F.col("kind")`). + """ + + items: list[_SyntheticUnionFixtures.SyntheticUnion] + + +class TestTopLevelListUnion: + """Field-level checks for `list[DiscriminatedUnion]` at the feature root. + + Regression test: the discriminator must be flagged as element-level so + the renderer accesses `el["kind"]` rather than `F.col("kind")`. + """ + + @pytest.fixture() + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(_ListUnionContainer) + return nodes + + def test_variant_field_uses_element_level_discriminator( + self, nodes: list[Check] + ) -> None: + for variant_field in ("a_field", "b_field"): + variant_nodes = [n for n in nodes if variant_field in str(n.target)] + assert variant_nodes, f"Expected variant-gated {variant_field} nodes" + for node in variant_nodes: + guard = _element_guard(node) + assert guard is not None, ( + f"{node.target}: list[Union] descendants must use the " + "element-level discriminator" + ) + assert guard.discriminator == "kind", ( + f"{node.target}: discriminator should be 'kind'" + ) + + +class _NestedListUnionContainer(BaseModel): + """Top-level `list[list[DiscriminatedUnion]]` with a constrained member. + + A union nested under multiple list layers would need the union + target to record `list_depth` iterations, but the rebase in + `_recurse_into_union` records only one. No real schema exercises + this path; `build_checks` raises rather than emit a target that + silently drops iterations. + """ + + nested: list[list[_SyntheticUnionFixtures.ConstrainedUnion]] + + +class TestNestedListUnionModelConstraints: + """`list[list[Union]]` raises rather than emit a collapsed target.""" + + def test_build_checks_raises_not_implemented(self) -> None: + with pytest.raises(NotImplementedError, match="multiple list layers"): + _checks_for(_NestedListUnionContainer) + + +class _DeepInnerModel(BaseModel): + value: Annotated[str, Field(min_length=1)] + + +class _DoubleNestedArrayModel(BaseModel): + items: list[list[_DeepInnerModel]] + + +class TestDoubleNestedArrayFieldChecks: + """Sub-field validation for list[list[Model]] (list_depth=2).""" + + @pytest.fixture() + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(_DoubleNestedArrayModel) + return nodes + + def test_subfield_target_encodes_both_array_levels( + self, nodes: list[Check] + ) -> None: + # A `list[list[Model]]` sub-field reaches `value` through a single + # ArraySegment with iter_count=2; the target pins the full geometry. + assert any(n.target == _path("items[][].value") for n in nodes) + + +class TestTripleNestedArrayFieldChecks: + """Verify depth=3 nesting generates correct geometry.""" + + @pytest.fixture() + def nodes(self) -> list[Check]: + nodes, _ = _checks_for(TripleNestedArrayModel) + return nodes + + def test_subfield_target_shows_three_brackets(self, nodes: list[Check]) -> None: + assert any(n.target == _path("deep[][][].tag") for n in nodes) + + +class _NestedScalarListModel(BaseModel): + """list[list[scalar]] terminating directly in a constrained scalar. + + Exercises the one nested-array geometry the other tests miss: an + element-level check whose target's terminal ArraySegment carries + iter_count > 1 with no struct leaf after it (`grid[][]`, not + `grid[][].field`). + """ + + grid: list[list[Annotated[str, MinLen(1)]]] + + +class TestNestedScalarListTarget: + """Element-level check on list[list[scalar]] targets a bare `field[][]`.""" + + def test_terminal_target_carries_iter_count_two(self) -> None: + nodes, _ = _checks_for(_NestedScalarListModel) + node = _node_for(nodes, "grid[][]", "check_string_min_length") + target = node.target + assert isinstance(target, ArrayPath) + last = target.segments[-1] + assert isinstance(last, ArraySegment) + assert last.name == "grid" + assert last.iter_count == 2 + + +class TestPrimitiveBoundsFiltered: + """Constraints inherent to primitive numeric types are filtered out.""" + + @pytest.fixture + def nodes(self) -> list[Check]: + """Field with int32-inherent and layered bounds.""" + shape = Primitive( + base_type="int32", + constraints=( + # Layered by schema author + ConstraintSource( + source_ref=None, source_name="FeatureVersion", constraint=Ge(ge=0) + ), + # Inherent to int32 + ConstraintSource( + source_ref=None, source_name="int32", constraint=Ge(ge=-(2**31)) + ), + ConstraintSource( + source_ref=None, source_name="int32", constraint=Le(le=2**31 - 1) + ), + ), + ) + field = FieldSpec( + name="version", shape=shape, description=None, is_required=True + ) + spec = RecordSpec(name="Test", description=None, fields=[field]) + nodes, _ = build_checks(spec) + return nodes + + def test_layered_bound_survives(self, nodes: list[Check]) -> None: + descs = nodes[0].descriptors + bounds = [d for d in descs if d.function == "check_bounds"] + assert len(bounds) == 1 + assert dict(bounds[0].kwargs) == {"ge": 0} + + def test_primitive_bounds_excluded(self, nodes: list[Check]) -> None: + descs = nodes[0].descriptors + bounds = [d for d in descs if d.function == "check_bounds"] + for b in bounds: + d = dict(b.kwargs) + assert d.get("ge") != -(2**31) + assert d.get("le") != 2**31 - 1 + + +@require_any_of("x", "y") +class _OptionalSubModelConstrained(BaseModel): + """Sub-model with require_any_of on its own fields.""" + + x: str | None = None + y: str | None = None + + +class _ElementWithOptionalConstrained(BaseModel): + nested: _OptionalSubModelConstrained | None = None + + +class _ArrayOfElementWithOptionalConstrained(BaseModel): + items: list[_ElementWithOptionalConstrained] + + +class TestOptionalSubModelModelCheckGate: + """ModelCheck for a constraint on an optional sub-model carries gate set to its path. + + When the constrained model is reached via an optional field (`field: Model | None`), + the PySpark validator must skip the constraint when the field is NULL. The + `ModelCheck.gate` carries the path to the optional field so the renderer can emit + `F.when(.isNotNull(), ...)`. + """ + + def test_optional_nested_model_gate_set(self) -> None: + """items[].nested is optional -- gate == path to nested.""" + _, model_nodes = _checks_for(_ArrayOfElementWithOptionalConstrained) + nodes = [ + n + for n in _filter_nodes(model_nodes, "check_require_any_of") + if n.target == _path("items[].nested") + ] + assert len(nodes) == 1 + assert nodes[0].gate == _path("items[].nested") + + def test_non_optional_sub_model_has_no_gate(self) -> None: + """Direct array element model (not optional) -- gate is None.""" + _, model_nodes = _checks_for(_ArrayOfConstrainedModel) + nodes = [ + n + for n in _filter_nodes(model_nodes, "check_require_any_of") + if n.target == _path("items[]") + ] + assert len(nodes) == 1 + assert nodes[0].gate is None + + def test_optional_list_field_element_model_has_no_gate(self) -> None: + """Optional list field (list[Model] | None) -- element constraint gate is None. + + The field being optional means the list itself may be absent; but the + constrained model is reached via array iteration, not a nullable struct + field, so no element-level gate belongs. + """ + _, model_nodes = _checks_for(_OptionalArrayOfConstrainedModel) + nodes = [ + n + for n in _filter_nodes(model_nodes, "check_require_any_of") + if n.target == _path("items[]") + ] + assert len(nodes) == 1 + assert nodes[0].gate is None + + def test_segment_speed_limits_when_has_gate(self) -> None: + """Segment.speed_limits[].when is optional -- gate == path to when.""" + spec = discover_feature("Segment") + _, model_nodes = build_checks(spec) + when_nodes = [ + n + for n in _filter_nodes(model_nodes, "check_require_any_of") + if n.target == _path("speed_limits[].when") + ] + assert len(when_nodes) >= 1 + for node in when_nodes: + assert node.gate == _path("speed_limits[].when") + + +class TestMapKeyValueConstraints: + """check_builder descends into MapOf key/value shapes. + + `FeatureWithDict.names` is `dict[LanguageTag, StrippedString]`: the key + carries `LanguageTagConstraint` (dispatches to check_pattern) and the + value carries `StrippedConstraint` (dispatches to check_stripped). Both + constraints are validated when the same NewTypes are reached through a + struct field -- generated transportation/segment.py emits check_pattern + for `names.rules[].language` -- so reaching them through a map must not + silently drop validation. + """ + + def _map_check(self, projection: MapProjection, function: str) -> Check: + field_checks, _ = _checks_for(FeatureWithDict) + matches = [ + c + for c in field_checks + if isinstance(c.target, MapPath) + and c.target.projection is projection + and any(d.function == function for d in c.descriptors) + ] + assert len(matches) >= 1, ( + f"no MapPath {projection} check with {function}; " + f"targets={[str(c.target) for c in field_checks]}" + ) + return matches[0] + + def test_map_key_pattern_check_targets_names_key(self) -> None: + check = self._map_check(MapProjection.KEY, "check_pattern") + assert str(check.target) == "names{key}" + + def test_map_value_stripped_check_targets_names_value(self) -> None: + check = self._map_check(MapProjection.VALUE, "check_stripped") + assert str(check.target) == "names{value}" + + def test_map_field_with_unconstrained_value_emits_no_value_check(self) -> None: + # metadata: dict[str, int] -- neither key nor value carries a + # constraint, so no MapPath checks are produced for it. + field_checks, _ = _checks_for(FeatureWithDict) + metadata_maps = [ + c + for c in field_checks + if isinstance(c.target, MapPath) and c.target.map_column == "metadata" + ] + assert metadata_maps == [] + + +class _MapWithConstrainedListValueModel(BaseModel): + """`dict[K, list[constrained-scalar]]` -- a map value carrying an array layer. + + `terminal_scalar` unwraps the `ArrayOf` to the inner scalar, so the + naive scalar guard lets this through; the value scalar's constraint + has no `MapPath` + `ArraySegment` geometry to land on. + """ + + items: dict[str, list[Annotated[str, MinLen(1)]]] + + +class _MapWithUnconstrainedListValueModel(BaseModel): + """`dict[K, list[scalar]]` with no key/value constraint -- nothing to emit.""" + + items: dict[str, list[int]] + + +class _ListOfConstrainedMapModel(BaseModel): + """`list[dict[K, constrained-scalar]]` -- a map reached through an array.""" + + items: list[dict[str, Annotated[str, MinLen(1)]]] + + +class _ListOfUnconstrainedMapModel(BaseModel): + """`list[dict[K, scalar]]` with no key/value constraint -- nothing to emit.""" + + items: list[dict[str, str]] + + +class _PlainScalarMapModel(BaseModel): + items: dict[str, str] + + +class _ConstrainedScalarMapModel(BaseModel): + items: dict[str, Annotated[str, MinLen(1)]] + + +class TestClassifyMapProjection: + """`classify_map_projection` is the single arbiter of map-shape support. + + Every map-shape prohibition in `_map_projection_checks` routes through + this classifier rather than restating the rule inline. The classifier + names the representable shape (struct-prefix -> one MapSegment -> scalar + or model/union terminal, reached without array iteration, no array layer + in the projected shape) and the reason each unsupported shape is rejected. + """ + + def _scalar_shape(self, *, constrained: bool) -> FieldShape: + spec = spec_for_model( + _ConstrainedScalarMapModel if constrained else _PlainScalarMapModel + ) + assert isinstance(spec, RecordSpec) + shape = spec.fields[0].shape + assert isinstance(shape, MapOf) + return shape.value + + def test_scalar_terminal_reached_struct_only_is_representable(self) -> None: + verdict = classify_map_projection( + self._scalar_shape(constrained=True), _path("items{value}") + ) + assert verdict.representable + assert verdict.reason is None + + def test_map_reached_through_array_is_rejected(self) -> None: + # The classifier owns the path-structural rejection too: a map_path + # that is an ArrayPath cannot anchor a struct-prefixed MapPath. + verdict = classify_map_projection( + self._scalar_shape(constrained=True), _path("items[]") + ) + assert not verdict.representable + assert verdict.reason is not None + + def test_array_layer_in_projected_shape_is_rejected(self) -> None: + spec = spec_for_model(_MapWithConstrainedListValueModel) + assert isinstance(spec, RecordSpec) + shape = spec.fields[0].shape + assert isinstance(shape, MapOf) + verdict = classify_map_projection(shape.value, _path("items{value}")) + assert not verdict.representable + assert verdict.reason is not None + + def test_classifier_rejects_dict_of_list_value(self) -> None: + # dict[K, list[V]]: the projected value shape carries an array layer. + # The classifier rejects it, and `_checks_for` raises -- the model + # raises iff the classifier rejects a shape with something to validate. + spec = spec_for_model(_MapWithConstrainedListValueModel) + assert isinstance(spec, RecordSpec) + shape = spec.fields[0].shape + assert isinstance(shape, MapOf) + verdict = classify_map_projection(shape.value, _path("items{value}")) + assert not verdict.representable + assert verdict.has_value_to_validate + with pytest.raises(NotImplementedError): + _checks_for(_MapWithConstrainedListValueModel) + + def test_classifier_rejects_map_reached_through_array(self) -> None: + # list[dict[K, V]]: the map is reached through an array, so the + # map_path is an ArrayPath. The classifier rejects on the path alone. + spec = spec_for_model(_ListOfConstrainedMapModel) + assert isinstance(spec, RecordSpec) + outer = spec.fields[0].shape + assert isinstance(outer, ArrayOf) + inner_map = outer.element + assert isinstance(inner_map, MapOf) + verdict = classify_map_projection(inner_map.value, _path("items[]")) + assert not verdict.representable + assert verdict.has_value_to_validate + with pytest.raises(NotImplementedError): + _checks_for(_ListOfConstrainedMapModel) + + +class TestMapProjectionUnsupportedShapes: + """`_map_projection_checks` is bounded to a scalar terminal reached struct-only. + + Two shapes fall outside that bound -- a map value/key with an array + layer (`dict[K, list[V]]`), and a map reached through an array + (`list[dict[K, V]]`). For each, a key/value constraint raises to keep + the dropped check loud, and an unconstrained one yields no checks (a + `MapPath` cannot locate the value, but there is nothing to validate). + """ + + def test_constrained_list_value_raises(self) -> None: + with pytest.raises(NotImplementedError, match="map value"): + _checks_for(_MapWithConstrainedListValueModel) + + def test_unconstrained_list_value_emits_no_projection_check(self) -> None: + field_checks, _ = _checks_for(_MapWithUnconstrainedListValueModel) + assert not any(isinstance(c.target, MapPath) for c in field_checks) + + def test_constrained_map_in_array_raises(self) -> None: + with pytest.raises(NotImplementedError, match="map value"): + _checks_for(_ListOfConstrainedMapModel) + + def test_unconstrained_map_in_array_emits_no_projection_check(self) -> None: + field_checks, _ = _checks_for(_ListOfUnconstrainedMapModel) + assert not any(isinstance(c.target, MapPath) for c in field_checks) + + +class _InnerLabel(BaseModel): + label: Annotated[str, MinLen(1)] + + +class _MapOfModel(BaseModel): + """A `dict[K, Model]` value model with a constrained scalar field. + + The value model's `label` field is validated on a `MapPath` leaf + (`items{value}.label`), the map analogue of a `list[Model]` element. + """ + + items: dict[str, _InnerLabel] + + +@require_any_of("foo", "bar") +class _AnyOfSub(BaseModel): + foo: int | None = None + bar: str | None = None + + +class _ModelConstraintAsMapValue(BaseModel): + """A `dict[K, Model]` value model carrying a model-level constraint. + + The `require_any_of` constraint is validated on the map value itself + (`subs{value}`). + """ + + subs: dict[str, _AnyOfSub] + + +class TestMapValueModelDescent: + """check_builder descends into a `dict[K, Model]` value model. + + A `ModelRef`/`UnionRef` map value is walked for its field and + model-level constraints on a `MapPath` target, the map analogue of a + `list[Model]` element reached through the `ModelRef` walker arm. + """ + + def test_value_field_constraint_targets_map_value_leaf(self) -> None: + field_checks, _ = _checks_for(_MapOfModel) + matches = [ + c + for c in field_checks + if isinstance(c.target, MapPath) + and str(c.target) == "items{value}.label" + and any(d.function == "check_string_min_length" for d in c.descriptors) + ] + assert len(matches) == 1, [str(c.target) for c in field_checks] + + def test_value_required_field_emits_required_descriptor(self) -> None: + field_checks, _ = _checks_for(_MapOfModel) + leaf_checks = [ + c + for c in field_checks + if isinstance(c.target, MapPath) and str(c.target) == "items{value}.label" + ] + assert leaf_checks + functions = {d.function for c in leaf_checks for d in c.descriptors} + assert "check_required" in functions + + def test_value_model_constraint_targets_map_value(self) -> None: + _, model_checks = _checks_for(_ModelConstraintAsMapValue) + matches = _filter_nodes(model_checks, "check_require_any_of", ("foo", "bar")) + assert len(matches) == 1 + assert isinstance(matches[0].target, MapPath) + assert str(matches[0].target) == "subs{value}" + + +class _MapValueWithList(BaseModel): + tags: list[Annotated[str, MinLen(1)]] + + +class _ListInsideMapValueModel(BaseModel): + """A `dict[K, Model]` value model with a constrained list field. + + A list nested inside a map element has no representable `MapPath`, so + the descent raises rather than emitting an unanchored target. + """ + + items: dict[str, _MapValueWithList] + + +class _UrlOrEmptyModel(BaseModel): + """Required field typed `HttpUrl | Literal[""]` -- literal bypass scenario.""" + + data_url: Annotated[HttpUrl | Literal[""], Field()] + + +class _OptionalCountryListModel(BaseModel): + """Optional list[CountryCodeAlpha2 | Literal["Global"]] -- array literal bypass.""" + + countries: ( + list[Annotated[CountryCodeAlpha2 | Literal["Global"], Field()]] | None + ) = None + + +class TestLiteralAlternativesBypass: + """check_builder threads allow_literals onto content descriptors for X | Literal[c] fields.""" + + def test_url_literal_bypass_on_content_descriptors(self) -> None: + """url_format and url_length carry allow_literals; check_required does not.""" + checks, _ = _checks_for(_UrlOrEmptyModel) + url_checks = [c for c in checks if str(c.target).endswith("data_url")] + assert url_checks, "expected a check targeting data_url" + # Required field: one Check with (check_required, url_format, url_length) + assert len(url_checks) == 1 + check = url_checks[0] + for desc in check.descriptors: + if desc.function == "check_required": + assert desc.allow_literals == (), ( + f"check_required must not carry allow_literals, got {desc.allow_literals}" + ) + else: + assert desc.allow_literals == ("",), ( + f"{desc.function} should carry allow_literals=('',), got {desc.allow_literals}" + ) + + def test_array_literal_bypass_on_element_descriptor(self) -> None: + """Array-element pattern check carries allow_literals for list[T | Literal[c]].""" + checks, _ = _checks_for(_OptionalCountryListModel) + country_checks = [c for c in checks if "countries" in str(c.target)] + assert country_checks, "expected a check targeting countries" + check = country_checks[0] + content_descs = [d for d in check.descriptors if d.function != "check_required"] + assert content_descs, "expected at least one content descriptor" + for desc in content_descs: + assert desc.allow_literals == ("Global",), ( + f"{desc.function} should carry allow_literals=('Global',), got {desc.allow_literals}" + ) + + def test_check_required_never_gets_allow_literals(self) -> None: + """check_required is excluded from the literal bypass even when coalesced.""" + checks, _ = _checks_for(_UrlOrEmptyModel) + for check in checks: + for desc in check.descriptors: + if desc.function == "check_required": + assert desc.allow_literals == (), ( + f"check_required at {check.target} carries unexpected allow_literals" + ) + + +class TestMapValueModelDescentBoundary: + """Descent raises where a `MapPath` cannot represent the shape. + + A map value model is descended into for scalar fields and model + constraints; a container (list or map) nested inside it has no + `MapPath` geometry, so the walker raises rather than emitting an + unvalidated target. + """ + + def test_list_inside_map_value_model_raises(self) -> None: + with pytest.raises(NotImplementedError, match="list nested inside a map"): + _checks_for(_ListInsideMapValueModel) diff --git a/packages/overture-schema-codegen/tests/test_pyspark_constraint_dispatch.py b/packages/overture-schema-codegen/tests/test_pyspark_constraint_dispatch.py new file mode 100644 index 000000000..765fdbb29 --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_pyspark_constraint_dispatch.py @@ -0,0 +1,616 @@ +"""Tests for pyspark constraint dispatch.""" + +import re + +import pytest +from annotated_types import Ge, Gt, Interval, Le, Lt +from overture.schema.codegen.extraction.field import Primitive +from overture.schema.codegen.extraction.length_constraints import ( + ArrayMaxLen, + ArrayMinLen, + ScalarMaxLen, + ScalarMinLen, +) +from overture.schema.codegen.extraction.literal_alternatives import LiteralAlternatives +from overture.schema.codegen.extraction.specs import FieldSpec +from overture.schema.codegen.pyspark.constraint_dispatch import ( + ExpressionDescriptor, + ForbidIf, + MinFieldsSet, + RadioGroup, + RequireAnyOf, + RequireAnyTrue, + RequireIf, + dispatch_base_type, + dispatch_constraint, + dispatch_model_constraint, + dispatch_newtype, + forbid_if_field_shapes, + model_constraint_function, + normalize_anchor, +) +from overture.schema.system.field_constraint.collection import UniqueItemsConstraint +from overture.schema.system.field_constraint.string import ( + CountryCodeAlpha2Constraint, + JsonPointerConstraint, + PatternConstraint, + SnakeCaseConstraint, + StrippedConstraint, +) +from overture.schema.system.model_constraint import ( + FieldEqCondition, + ForbidIfConstraint, + MinFieldsSetConstraint, + NoExtraFieldsConstraint, + Not, + RadioGroupConstraint, + RequireAnyOfConstraint, + RequireAnyTrueConstraint, + RequireIfConstraint, +) +from overture.schema.system.primitive import GeometryType, GeometryTypeConstraint +from overture.schema.system.ref import Identified, Reference, Relationship +from pydantic import Field, Strict + + +class _Stub(Identified): + pass + + +class TestBoundsDispatch: + @pytest.mark.parametrize( + ("constraint", "expected_kwargs"), + [ + (Ge(ge=0), (("ge", 0),)), + (Gt(gt=0), (("gt", 0),)), + (Le(le=100), (("le", 100),)), + (Lt(lt=100), (("lt", 100),)), + (Interval(ge=0, le=1), (("ge", 0), ("le", 1))), + (Interval(ge=0), (("ge", 0),)), + ], + ) + def test_bound_dispatches_to_check_bounds( + self, constraint: object, expected_kwargs: tuple[tuple[str, object], ...] + ) -> None: + desc = dispatch_constraint(constraint) + assert desc is not None + assert desc.function == "check_bounds" + assert desc.kwargs == expected_kwargs + + def test_int_bounds_coerced_to_float_for_float_type(self) -> None: + """Integer bound values become float when the field is a float type.""" + desc = dispatch_constraint(Ge(ge=0), base_type="float64") + assert desc is not None + assert desc.kwargs == (("ge", 0.0),) + assert isinstance(dict(desc.kwargs)["ge"], float) + + def test_int_bounds_preserved_for_int_type(self) -> None: + desc = dispatch_constraint(Ge(ge=0), base_type="int32") + assert desc is not None + assert desc.kwargs == (("ge", 0),) + assert isinstance(dict(desc.kwargs)["ge"], int) + + def test_float_bounds_unchanged_for_float_type(self) -> None: + desc = dispatch_constraint(Ge(ge=0.0), base_type="float64") + assert desc is not None + assert desc.kwargs == (("ge", 0.0),) + assert isinstance(dict(desc.kwargs)["ge"], float) + + def test_float_bound_sets_check_nan_none(self) -> None: + """Float-typed bounds leave check_nan unset (runtime defaults to guarded).""" + desc = dispatch_constraint(Ge(ge=0), base_type="float64") + assert desc is not None + assert desc.check_nan is None + + def test_integer_bound_sets_check_nan_false(self) -> None: + """Integer-typed bounds set check_nan=False to skip the dead NaN guard.""" + desc = dispatch_constraint(Ge(ge=0), base_type="int32") + assert desc is not None + assert desc.check_nan is False + + def test_untyped_bound_sets_check_nan_none(self) -> None: + """Bounds without a base_type leave check_nan unset (safe default).""" + desc = dispatch_constraint(Ge(ge=0)) + assert desc is not None + assert desc.check_nan is None + + def test_kwargs_contains_only_bounds(self) -> None: + """check_nan does not appear in kwargs; only ge/gt/le/lt keys are present. + + Uses Interval(ge=0, le=1) so the descriptor has two bound kwargs, + making the assertion non-vacuous: a stray non-bound kwarg alongside + a real bound would cause the check to fail. + """ + desc = dispatch_constraint(Interval(ge=0, le=1), base_type="int32") + assert desc is not None + kwarg_keys = {k for k, _ in desc.kwargs} + assert kwarg_keys == {"ge", "le"} + assert kwarg_keys <= {"ge", "gt", "le", "lt"} + + +class TestLengthDispatch: + def test_min_len_on_array(self) -> None: + desc = dispatch_constraint(ArrayMinLen(min_length=2)) + assert desc == ExpressionDescriptor( + function="check_array_min_length", args=(2,) + ) + + def test_min_len_on_scalar(self) -> None: + desc = dispatch_constraint(ScalarMinLen(min_length=1)) + assert desc == ExpressionDescriptor( + function="check_string_min_length", args=(1,) + ) + + def test_max_len_on_array(self) -> None: + desc = dispatch_constraint(ArrayMaxLen(max_length=10)) + assert desc == ExpressionDescriptor( + function="check_array_max_length", args=(10,) + ) + + def test_max_len_on_scalar(self) -> None: + desc = dispatch_constraint(ScalarMaxLen(max_length=10)) + assert desc == ExpressionDescriptor( + function="check_string_max_length", args=(10,) + ) + + +class TestStringConstraintDispatch: + def test_stripped(self) -> None: + desc = dispatch_constraint(StrippedConstraint()) + assert desc is not None + assert desc.function == "check_stripped" + assert desc.constraint_type is StrippedConstraint + + def test_json_pointer(self) -> None: + desc = dispatch_constraint(JsonPointerConstraint()) + assert desc is not None + assert desc.function == "check_json_pointer" + assert desc.constraint_type is JsonPointerConstraint + + def test_pattern_constraint_base(self) -> None: + c = PatternConstraint(r"^[A-Z]{2}$", "test error") + desc = dispatch_constraint(c) + assert desc is not None + assert desc.function == "check_pattern" + assert desc.args == (r"^[A-Z]{2}\z",) # anchor normalized + + def test_country_code_dispatches_as_pattern(self) -> None: + c = CountryCodeAlpha2Constraint() + desc = dispatch_constraint(c) + assert desc is not None + assert desc.function == "check_pattern" + assert desc.args == (r"^[A-Z]{2}\z",) # anchor normalized + assert desc.label == "ISO 3166-1 alpha-2 country code" + assert desc.check_name == "country_code_alpha2" + + def test_snake_case_dispatches_as_pattern(self) -> None: + c = SnakeCaseConstraint() + desc = dispatch_constraint(c) + assert desc is not None + assert desc.function == "check_pattern" + assert desc.args == (r"^[a-z0-9]+(_[a-z0-9]+)*\z",) # anchor normalized + assert desc.label == "Category in snake_case format" + assert desc.check_name == "snake_case" + + +class TestRawPydanticPatternDispatch: + """Raw pydantic `Field(pattern=)` metadata (`_PydanticGeneralMetadata`). + + Distinguished from the schema's `PatternConstraint` by being a + `PydanticMetadata` marker. Carries the pattern as a `str` + (`Field(pattern="...")`) or a compiled `re.Pattern` + (`Field(pattern=re.compile(...))` -- the only flagged-pattern carrier). + Reaches dispatch via map keys today (e.g. `Sources.license_priority`). + """ + + def test_pydantic_pattern_metadata_dispatches_as_pattern(self) -> None: + (meta,) = Field(pattern=r"^[a-z]+$").metadata + desc = dispatch_constraint(meta) + assert desc is not None + assert desc.function == "check_pattern" + assert desc.args == (r"^[a-z]+\z",) # anchor-normalized + + def test_compiled_pattern_metadata_dispatches_as_pattern(self) -> None: + # A compiled re.Pattern is the only carrier for a flagged pattern, so + # `Field(pattern=re.compile(...))` must dispatch like a bare string. + (meta,) = Field(pattern=re.compile(r"^[a-z]+$")).metadata + desc = dispatch_constraint(meta) + assert desc is not None + assert desc.function == "check_pattern" + assert desc.args == (r"^[a-z]+\z",) # anchor-normalized + + def test_compiled_pattern_ignorecase_prepends_inline_flag(self) -> None: + # re.IGNORECASE has no string-pattern carrier; it maps to Spark's + # inline (?i) flag (the same idiom check_url_format uses). + (meta,) = Field(pattern=re.compile(r"^[a-z]+$", re.I)).metadata + desc = dispatch_constraint(meta) + assert desc is not None + assert desc.function == "check_pattern" + assert desc.args == (r"(?i)^[a-z]+\z",) + + def test_compiled_pattern_unsupported_flag_raises_named(self) -> None: + # An untranslatable flag must raise a clean, flag-naming error rather + # than the opaque "Unhandled constraint type" TypeError. + (meta,) = Field(pattern=re.compile(r"^[a-z]+$", re.M)).metadata + with pytest.raises(NotImplementedError, match="MULTILINE"): + dispatch_constraint(meta) + + def test_plain_object_with_str_pattern_still_raises(self) -> None: + # A non-PydanticMetadata object that merely exposes a string + # `.pattern` must not be mistaken for raw pattern metadata: the + # fallback contract stays "raise on unhandled", so an unrelated + # future constraint can't be silently turned into a check_pattern. + class _Imposter: + pattern = r"^[a-z]+$" + + with pytest.raises(TypeError, match="Unhandled constraint type"): + dispatch_constraint(_Imposter()) + + def test_non_pattern_object_still_raises(self) -> None: + class _Unknown: + pass + + with pytest.raises(TypeError, match="Unhandled constraint type"): + dispatch_constraint(_Unknown()) + + +class TestPatternConstraintDispatch: + def test_pattern_constraint_label_fallback_to_docstring(self) -> None: + """PatternConstraint with no description falls back to docstring, period stripped.""" + c = PatternConstraint(r"^test$", "error: {value}") + desc = dispatch_constraint(c) + assert desc is not None + # Base PatternConstraint has docstring "Generic pattern-based string constraint." + assert desc.label == "Generic pattern-based string constraint" + + def test_pattern_constraint_check_name_base_class(self) -> None: + c = PatternConstraint(r"^test$", "error: {value}") + desc = dispatch_constraint(c) + assert desc is not None + assert desc.check_name == "pattern" + + def test_anchor_normalized_dollar_to_backslash_z(self) -> None: + c = CountryCodeAlpha2Constraint() # pattern ends with $ + desc = dispatch_constraint(c) + assert desc is not None + pattern = str(desc.args[0]) + assert pattern.endswith(r"\z") + assert not pattern.endswith("$") + + def test_anchor_normalization_replaces_only_trailing_dollar(self) -> None: + """Dollar signs inside character classes are not end-anchors.""" + c = PatternConstraint(r"^[\$]+$", "error: {value}") + desc = dispatch_constraint(c) + assert desc is not None + pattern = str(desc.args[0]) + # The trailing $ is replaced; the \$ inside the class is preserved + assert pattern == r"^[\$]+\z" + + def test_ignorecase_flag_prepends_inline_flag(self) -> None: + """A case-insensitive PatternConstraint maps re.I to Spark's (?i).""" + c = PatternConstraint(r"^[a-z]+$", "error: {value}", flags=re.I) + desc = dispatch_constraint(c) + assert desc is not None + assert desc.args == (r"(?i)^[a-z]+\z",) + + def test_unsupported_flag_raises_named(self) -> None: + """An untranslatable flag raises a clean, flag-naming error.""" + c = PatternConstraint(r"^[a-z]+$", "error: {value}", flags=re.M) + with pytest.raises(NotImplementedError, match="MULTILINE"): + dispatch_constraint(c) + + +class TestStructuralConstraintDispatch: + def test_unique_items(self) -> None: + desc = dispatch_constraint(UniqueItemsConstraint()) + assert desc is not None + assert desc.function == "check_struct_unique" + + def test_geometry_type(self) -> None: + c = GeometryTypeConstraint(GeometryType.POINT) + desc = dispatch_constraint(c) + assert desc is not None + assert desc.function == "check_geometry_type" + assert GeometryType.POINT in desc.args + + +class TestSkippedConstraints: + def test_reference_returns_none(self) -> None: + r = Reference(Relationship.AGGREGATION, _Stub) + desc = dispatch_constraint(r) + assert desc is None + + def test_strict_returns_none(self) -> None: + desc = dispatch_constraint(Strict()) + assert desc is None + + def test_literal_alternatives_returns_none(self) -> None: + # The literal-alternatives bypass is a modifier on the field's other + # checks (threaded as allow_literals), not a standalone check. + desc = dispatch_constraint(LiteralAlternatives(("",))) + assert desc is None + + +class TestBaseTypeDispatch: + def test_http_url_dispatches_to_check_url_format_and_length(self) -> None: + descs = dispatch_base_type("HttpUrl") + assert descs is not None + assert len(descs) == 2 + assert descs[0].function == "check_url_format" + assert descs[1].function == "check_url_length" + + def test_email_str_dispatches_to_check_email(self) -> None: + descs = dispatch_base_type("EmailStr") + assert descs is not None + assert len(descs) == 1 + assert descs[0].function == "check_email" + + def test_bbox_dispatches_to_three_checks(self) -> None: + descs = dispatch_base_type("BBox") + assert descs is not None + assert len(descs) == 3 + assert descs[0].function == "check_bbox_completeness" + assert descs[1].function == "check_bbox_lat_ordering" + assert descs[2].function == "check_bbox_lat_range" + + def test_unknown_base_type_returns_none(self) -> None: + descs = dispatch_base_type("str") + assert descs is None + + +class TestNewtypeDispatch: + def test_linear_range(self) -> None: + descs = dispatch_newtype("LinearlyReferencedRange") + assert descs is not None + assert len(descs) == 3 + assert descs[0].function == "check_linear_range_length" + assert descs[1].function == "check_linear_range_bounds" + assert descs[2].function == "check_linear_range_order" + + def test_country_code_alpha2_returns_none(self) -> None: + descs = dispatch_newtype("CountryCodeAlpha2") + assert descs is None + + def test_region_code_returns_none(self) -> None: + descs = dispatch_newtype("RegionCode") + assert descs is None + + def test_unknown_newtype_returns_none(self) -> None: + desc = dispatch_newtype("FeatureVersion") + assert desc is None + + +class TestPatternLabelAcronymHandling: + def test_acronym_run_in_name_splits_correctly(self) -> None: + """PatternConstraint subclass with an acronym run labels with spaces.""" + + class JSONPathConstraint(PatternConstraint): + def __init__(self) -> None: + super().__init__(r"^\$", "Invalid JSON path: {value}") + + desc = dispatch_constraint(JSONPathConstraint()) + assert desc is not None + assert desc.label == "json path" + + +class TestUnknownConstraintFails: + def test_unknown_constraint_raises(self) -> None: + with pytest.raises(TypeError, match="Unhandled constraint"): + dispatch_constraint(object()) + + +class TestModelConstraintDispatch: + def test_require_any_of(self) -> None: + c = RequireAnyOfConstraint("a", "b") + (desc,) = dispatch_model_constraint(c, []) + assert isinstance(desc, RequireAnyOf) + assert model_constraint_function(desc) == "check_require_any_of" + assert desc.field_names == ("a", "b") + + def test_require_any_true(self) -> None: + c = RequireAnyTrueConstraint( + FieldEqCondition(field_name="is_land", value=True), + FieldEqCondition(field_name="is_territorial", value=True), + ) + (desc,) = dispatch_model_constraint(c, []) + assert isinstance(desc, RequireAnyTrue) + assert model_constraint_function(desc) == "check_require_any_true" + assert desc.conditions == c.conditions + + def test_require_any_true_rejects_negated_condition(self) -> None: + """PySpark support is positive-only; a negated condition fails loudly. + + The runtime coalesces a null condition to False, which matches a + positive equality but flips a negated one on a null field, so + dispatch refuses to lower it rather than emit a wrong check. + """ + c = RequireAnyTrueConstraint( + Not(FieldEqCondition(field_name="is_land", value=True)), + ) + with pytest.raises(TypeError, match="positive boolean FieldEqConditions"): + dispatch_model_constraint(c, []) + + def test_require_any_true_rejects_non_boolean_condition(self) -> None: + """PySpark support is boolean-flag-only. + + The test-data disabling value is the boolean's negation, so a + non-boolean equality (e.g. `subtype == "county"`) fails loudly at + dispatch rather than later, mid test-module generation. + """ + c = RequireAnyTrueConstraint( + FieldEqCondition(field_name="subtype", value="county"), + ) + with pytest.raises(TypeError, match="positive boolean FieldEqConditions"): + dispatch_model_constraint(c, []) + + def test_radio_group(self) -> None: + c = RadioGroupConstraint("is_land", "is_territorial") + (desc,) = dispatch_model_constraint(c, []) + assert isinstance(desc, RadioGroup) + assert model_constraint_function(desc) == "check_radio_group" + assert desc.field_names == ("is_land", "is_territorial") + + def test_require_if(self) -> None: + c = RequireIfConstraint( + field_names=("class",), + condition=FieldEqCondition(field_name="subtype", value="road"), + ) + (desc,) = dispatch_model_constraint(c, []) + assert isinstance(desc, RequireIf) + assert model_constraint_function(desc) == "check_require_if" + assert desc.field_names == ("class",) + assert desc.condition is c.condition + + def test_require_if_multi_field_splits(self) -> None: + """Multi-field `@require_if(["a", "b"], cond)` splits into one descriptor per field. + + Each runtime `check_require_if` call takes a single target + column, so the descriptor mirrors that: one per field, sharing + the same condition. + """ + condition = FieldEqCondition(field_name="subtype", value="road") + c = RequireIfConstraint(field_names=("a", "b"), condition=condition) + descs = dispatch_model_constraint(c, []) + assert len(descs) == 2 + assert all(isinstance(d, RequireIf) for d in descs) + assert [d.field_names for d in descs] == [("a",), ("b",)] + assert all(d.condition is condition for d in descs) # type: ignore[union-attr] + + def test_forbid_if(self) -> None: + c = ForbidIfConstraint( + field_names=("class",), + condition=FieldEqCondition(field_name="subtype", value="water"), + ) + (desc,) = dispatch_model_constraint(c, []) + assert isinstance(desc, ForbidIf) + assert model_constraint_function(desc) == "check_forbid_if" + assert desc.field_names == ("class",) + assert desc.field_shapes == () + + def test_forbid_if_negated(self) -> None: + c = ForbidIfConstraint( + field_names=("parent_division_id",), + condition=Not(FieldEqCondition(field_name="subtype", value="country")), + ) + (desc,) = dispatch_model_constraint(c, []) + assert isinstance(desc, ForbidIf) + assert model_constraint_function(desc) == "check_forbid_if" + assert desc.condition is c.condition + + def test_forbid_if_multi_field_splits(self) -> None: + """Multi-field `@forbid_if` splits into one descriptor per field, each with its own shape.""" + condition = FieldEqCondition(field_name="subtype", value="road") + c = ForbidIfConstraint(field_names=("a", "b"), condition=condition) + descs = dispatch_model_constraint(c, []) + assert len(descs) == 2 + assert all(isinstance(d, ForbidIf) for d in descs) + assert [d.field_names for d in descs] == [("a",), ("b",)] + + def test_min_fields_set(self) -> None: + c = MinFieldsSetConstraint(count=1) + (desc,) = dispatch_model_constraint(c, []) + assert isinstance(desc, MinFieldsSet) + assert model_constraint_function(desc) == "check_min_fields_set" + assert desc.count == 1 + assert desc.field_names == () + + def test_min_fields_set_enumerates_all_fields(self) -> None: + """`field_names` holds every field -- required and optional alike. + + Matches Pydantic's `model_fields_set` semantics, where required + fields are always set by the constructor and contribute to the + count alongside any explicitly-set optional fields. + """ + fields = [ + FieldSpec(name="required_a", shape=Primitive(base_type="str")), + FieldSpec( + name="optional_b", + shape=Primitive(base_type="str"), + is_required=False, + ), + FieldSpec(name="required_c", shape=Primitive(base_type="str")), + FieldSpec( + name="optional_d", + shape=Primitive(base_type="str"), + is_required=False, + ), + ] + c = MinFieldsSetConstraint(count=1) + (desc,) = dispatch_model_constraint(c, fields) + assert isinstance(desc, MinFieldsSet) + assert desc.field_names == ( + "required_a", + "optional_b", + "required_c", + "optional_d", + ) + + def test_no_extra_fields_skipped(self) -> None: + c = NoExtraFieldsConstraint() + assert dispatch_model_constraint(c, []) == () + + def test_unknown_model_constraint_raises(self) -> None: + with pytest.raises(TypeError, match="Unhandled model constraint"): + dispatch_model_constraint(object(), []) + + +class TestForbidIfFieldShapes: + """Non-string scalar shapes must appear in field_shapes.""" + + @pytest.mark.parametrize( + ("base_type", "field_name"), + [ + ("int32", "count"), + ("bool", "flag"), + ("float64", "score"), + ], + ) + def test_non_string_scalar_included_in_field_shapes( + self, base_type: str, field_name: str + ) -> None: + shape = Primitive(base_type=base_type) + result = forbid_if_field_shapes((field_name,), {field_name: shape}) + assert len(result) == 1 + assert result[0][0] == field_name + + def test_string_scalar_excluded_from_field_shapes(self) -> None: + """String scalars remain excluded; renderer defaults to '' fill.""" + shape = Primitive(base_type="str") + result = forbid_if_field_shapes(("label",), {"label": shape}) + assert result == () + + def test_dispatch_model_constraint_forbid_if_int_has_field_shapes(self) -> None: + condition = FieldEqCondition(field_name="subtype", value="road") + c = ForbidIfConstraint(field_names=("version",), condition=condition) + fields = [ + FieldSpec(name="version", shape=Primitive(base_type="int32")), + ] + (desc,) = dispatch_model_constraint(c, fields) + assert isinstance(desc, ForbidIf) + assert len(desc.field_shapes) == 1 + assert desc.field_shapes[0][0] == "version" + + +class TestNormalizeAnchorParity: + """normalize_anchor uses backslash-parity to distinguish anchor from escaped $.""" + + def test_bare_dollar_converted(self) -> None: + assert normalize_anchor(r"foo$") == r"foo\z" + + def test_escaped_dollar_left_unchanged(self) -> None: + """Single backslash before $ -- literal dollar, must not convert.""" + assert normalize_anchor(r"foo\$") == r"foo\$" + + def test_escaped_backslash_then_anchor_converted(self) -> None: + """Two backslashes before $ -- even parity, $ is a real anchor, must convert.""" + # "foo\\\\$" is the 6-char string: f o o \ \ $ + # Even number of backslashes (2) before $: the $ is an unescaped anchor. + result = normalize_anchor("foo\\\\$") + assert result.endswith(r"\z"), f"Expected \\\\z suffix, got {result!r}" + assert not result.endswith("$") + + def test_triple_backslash_dollar_left_unchanged(self) -> None: + r"""Three backslashes before $ -- odd parity, $ is a literal dollar.""" + # "foo\\\\\\$" -- three backslashes + $, odd count: escaped literal $ + result = normalize_anchor("foo\\\\\\$") + assert result.endswith("$"), f"Expected trailing $, got {result!r}" diff --git a/packages/overture-schema-codegen/tests/test_pyspark_constraint_values.py b/packages/overture-schema-codegen/tests/test_pyspark_constraint_values.py new file mode 100644 index 000000000..419e2866e --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_pyspark_constraint_values.py @@ -0,0 +1,231 @@ +"""Tests for the paired constraint value table.""" + +import pytest +from overture.schema.codegen.pyspark.constraint_dispatch import ExpressionDescriptor +from overture.schema.codegen.pyspark.test_data.constraint_values import ( + CONSTRAINT_VALUES, + invalid_bound, + valid_bound, +) +from overture.schema.system.field_constraint.string import ( + CountryCodeAlpha2Constraint, + HexColorConstraint, + JsonPointerConstraint, + LanguageTagConstraint, + NoWhitespaceConstraint, + PatternConstraint, + PhoneNumberConstraint, + RegionCodeConstraint, + SnakeCaseConstraint, + StrippedConstraint, + WikidataIdConstraint, +) + + +class TestConstraintValuesCompleteness: + """CONSTRAINT_VALUES covers the expected set of constraint types.""" + + def test_expected_constraint_types_present(self) -> None: + expected = { + CountryCodeAlpha2Constraint, + HexColorConstraint, + JsonPointerConstraint, + LanguageTagConstraint, + NoWhitespaceConstraint, + PhoneNumberConstraint, + RegionCodeConstraint, + SnakeCaseConstraint, + StrippedConstraint, + WikidataIdConstraint, + } + assert expected <= set(CONSTRAINT_VALUES.keys()) + + +def _pattern_entries() -> list[type]: + # All CONSTRAINT_VALUES keys that are PatternConstraint subclasses, minus those + # with dedicated behavioural tests. + # StrippedConstraint IS a PatternConstraint subclass but uses \Z (not portable + # as a regex literal), so its contract is verified in TestStrippedConstraintValues. + # JsonPointerConstraint is NOT a PatternConstraint subclass — it has no .pattern + # attribute — and is verified in TestJsonPointerConstraintValues. + _BEHAVIOURAL_EXCLUSIONS = {StrippedConstraint, JsonPointerConstraint} + return sorted( + [ + ct + for ct in CONSTRAINT_VALUES + if issubclass(ct, PatternConstraint) and ct not in _BEHAVIOURAL_EXCLUSIONS + ], + key=lambda ct: ct.__name__, + ) + + +class TestPatternConstraintValues: + """For each PatternConstraint subclass, the valid value matches and invalid does not.""" + + _PATTERN_ENTRIES = _pattern_entries() + + @pytest.mark.parametrize( + "constraint_type", _PATTERN_ENTRIES, ids=lambda ct: ct.__name__ + ) + def test_valid_matches_pattern(self, constraint_type: type) -> None: + constraint = constraint_type() + assert isinstance(constraint, PatternConstraint) + cv = CONSTRAINT_VALUES[constraint_type] + assert isinstance(cv.valid, str) + assert constraint.pattern.match(cv.valid), ( + f"{constraint_type.__name__}: valid value {cv.valid!r} " + f"did not match pattern {constraint.pattern.pattern!r}" + ) + + @pytest.mark.parametrize( + "constraint_type", _PATTERN_ENTRIES, ids=lambda ct: ct.__name__ + ) + def test_invalid_does_not_match_pattern(self, constraint_type: type) -> None: + constraint = constraint_type() + assert isinstance(constraint, PatternConstraint) + cv = CONSTRAINT_VALUES[constraint_type] + assert isinstance(cv.invalid, str) + assert not constraint.pattern.match(cv.invalid), ( + f"{constraint_type.__name__}: invalid value {cv.invalid!r} " + f"matched pattern {constraint.pattern.pattern!r} (should not)" + ) + + +class TestStrippedConstraintValues: + """StrippedConstraint valid/invalid contract verified behaviorally.""" + + def test_valid_is_stripped(self) -> None: + cv = CONSTRAINT_VALUES[StrippedConstraint] + assert isinstance(cv.valid, str) + assert cv.valid == cv.valid.strip() + + def test_invalid_has_leading_or_trailing_whitespace(self) -> None: + cv = CONSTRAINT_VALUES[StrippedConstraint] + assert isinstance(cv.invalid, str) + assert cv.invalid != cv.invalid.strip() + + +class TestJsonPointerConstraintValues: + """JsonPointerConstraint valid/invalid contract verified behaviorally.""" + + def test_valid_starts_with_slash_or_is_empty(self) -> None: + cv = CONSTRAINT_VALUES[JsonPointerConstraint] + assert isinstance(cv.valid, str) + assert cv.valid == "" or cv.valid.startswith("/") + + def test_invalid_does_not_start_with_slash(self) -> None: + cv = CONSTRAINT_VALUES[JsonPointerConstraint] + assert isinstance(cv.invalid, str) + assert cv.invalid != "" and not cv.invalid.startswith("/") + + +class TestBoundFunctions: + """valid_bound and invalid_bound produce values on opposite sides of each bound kind.""" + + def test_valid_bound_ge(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("ge", 5),)) + assert valid_bound(desc) == 5 + + def test_valid_bound_gt(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("gt", 5),)) + assert valid_bound(desc) == 6 + + def test_valid_bound_le(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("le", 5),)) + assert valid_bound(desc) == 5 + + def test_valid_bound_lt(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("lt", 5),)) + assert valid_bound(desc) == 4 + + def test_valid_bound_fallback_to_zero(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=()) + assert valid_bound(desc) == 0 + + def test_invalid_bound_ge(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("ge", 5),)) + assert invalid_bound(desc) == 4 + + def test_invalid_bound_gt(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("gt", 5),)) + assert invalid_bound(desc) == 5 + + def test_invalid_bound_le(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("le", 5),)) + assert invalid_bound(desc) == 6 + + def test_invalid_bound_lt(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("lt", 5),)) + assert invalid_bound(desc) == 5 + + def test_invalid_bound_unknown_raises(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("unknown", 5),)) + with pytest.raises(ValueError): + invalid_bound(desc) + + def test_valid_bound_gt_lt_float_in_range(self) -> None: + """Interval(gt=0.0, lt=0.5) returns a value strictly between 0.0 and 0.5.""" + desc = ExpressionDescriptor( + function="check_bounds", kwargs=(("gt", 0.0), ("lt", 0.5)) + ) + result = valid_bound(desc) + assert isinstance(result, (int, float)) + assert 0 < result < 0.5 + + def test_valid_bound_gt_lt_int_non_degenerate(self) -> None: + """Adjacent-but-valid int intervals return the interior midpoint.""" + desc_2 = ExpressionDescriptor( + function="check_bounds", kwargs=(("gt", 0), ("lt", 2)) + ) + assert valid_bound(desc_2) == 1 + desc_4 = ExpressionDescriptor( + function="check_bounds", kwargs=(("gt", 0), ("lt", 4)) + ) + assert valid_bound(desc_4) == 2 + + def test_valid_bound_gt_lt_int_degenerate_raises(self) -> None: + """Adjacent exclusive int bounds (gt=0, lt=1) have no valid integer midpoint.""" + desc = ExpressionDescriptor( + function="check_bounds", kwargs=(("gt", 0), ("lt", 1)) + ) + with pytest.raises(ValueError, match="gt=0"): + valid_bound(desc) + + def test_valid_bound_ge_le_in_range(self) -> None: + """Interval(ge=0, le=10) returns a value in [0, 10].""" + desc = ExpressionDescriptor( + function="check_bounds", kwargs=(("ge", 0), ("le", 10)) + ) + result = valid_bound(desc) + assert isinstance(result, (int, float)) + assert 0 <= result <= 10 + + def test_valid_bound_gt_float_returns_float(self) -> None: + """gt=0.5 (float bound) returns a float value > 0.5.""" + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("gt", 0.5),)) + result = valid_bound(desc) + assert isinstance(result, float) + assert result > 0.5 + + def test_valid_bound_lt_float_returns_float(self) -> None: + """lt=0.5 (float bound) returns a float value < 0.5.""" + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("lt", 0.5),)) + result = valid_bound(desc) + assert isinstance(result, float) + assert result < 0.5 + + def test_valid_bound_gt_lt_float_tight_interval(self) -> None: + """gt=0.5, lt=1.0: midpoint 0.75 satisfies both bounds.""" + desc = ExpressionDescriptor( + function="check_bounds", kwargs=(("gt", 0.5), ("lt", 1.0)) + ) + result = valid_bound(desc) + assert isinstance(result, float) + assert 0.5 < result < 1.0 + + def test_invalid_bound_confirmed_correct(self) -> None: + """invalid_bound is already correct — one violated bound suffices.""" + ge_desc = ExpressionDescriptor(function="check_bounds", kwargs=(("ge", 3),)) + assert invalid_bound(ge_desc) == 2 + lt_desc = ExpressionDescriptor(function="check_bounds", kwargs=(("lt", 3),)) + assert invalid_bound(lt_desc) == 3 diff --git a/packages/overture-schema-codegen/tests/test_pyspark_e2e.py b/packages/overture-schema-codegen/tests/test_pyspark_e2e.py new file mode 100644 index 000000000..6ec7f663b --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_pyspark_e2e.py @@ -0,0 +1,210 @@ +"""End-to-end generation tests: verify generated modules match hand-written references.""" + +import ast +from pathlib import Path +from typing import Annotated, Literal + +import pytest +from annotated_types import Ge +from codegen_test_support import discover_feature +from overture.schema.codegen.cli import _generate_pyspark +from overture.schema.codegen.extraction.model_extraction import extract_model +from overture.schema.codegen.pyspark.pipeline import ( + GeneratedModule, + generate_pyspark_module, +) +from pydantic import BaseModel + + +class SimpleModel(BaseModel): + subtype: Literal["a", "b"] + score: Annotated[float, Ge(0.0)] | None = None + + +class TestDivisionAreaGeneration: + @pytest.fixture + def generated(self) -> GeneratedModule: + spec = discover_feature("DivisionArea") + return generate_pyspark_module(spec) + + def test_generates_valid_python(self, generated: GeneratedModule) -> None: + ast.parse(generated.content) + + def test_has_builder_function(self, generated: GeneratedModule) -> None: + assert "def division_area_checks()" in generated.content + + def test_has_schema_constant(self, generated: GeneratedModule) -> None: + assert "DIVISION_AREA_SCHEMA" in generated.content + + def test_output_path(self, generated: GeneratedModule) -> None: + assert generated.path.name == "division_area.py" + + def test_checks_cover_expected_fields(self, generated: GeneratedModule) -> None: + """Generated checks should cover the fields from the hand-written module.""" + content = generated.content + # Hand-written checks: subtype, class, country, region, require_any_true (is_land, is_territorial), admin_level + for field in ["subtype", "class", "country", "region"]: + assert f"field='{field}'" in content, f"Missing check for {field}" + + def test_schema_has_expected_fields(self, generated: GeneratedModule) -> None: + """Schema should contain all expected DivisionArea fields.""" + content = generated.content + expected_fields = [ + "id", + "geometry", + "bbox", + "country", + "version", + "subtype", + "class", + "names", + "is_land", + "is_territorial", + "region", + "admin_level", + "division_id", + "theme", + "type", + ] + for field in expected_fields: + assert f'"{field}"' in content, f"Missing schema field: {field}" + + def test_uses_bbox_shared_struct(self, generated: GeneratedModule) -> None: + """Should reference BBOX_STRUCT from _schema_structs (BBox is not a BaseModel).""" + assert "BBOX_STRUCT" in generated.content + + def test_imports_constraint_expressions(self, generated: GeneratedModule) -> None: + """Should import constraint expression functions.""" + content = generated.content + assert ( + "from overture.schema.pyspark.expressions.constraint_expressions import" + in content + ) + + def test_require_any_true_constraint(self, generated: GeneratedModule) -> None: + """Should have a require_any_true check for is_land/is_territorial. + + DivisionArea uses `@require_any_true(FieldEqCondition("is_land", True), + FieldEqCondition("is_territorial", True))` -- at least one flag true, + both-true allowed (unlike radio_group's exactly-one). + """ + content = generated.content + assert "check_require_any_true" in content + assert "is_land" in content + assert "is_territorial" in content + + def test_subtype_has_check_enum(self, generated: GeneratedModule) -> None: + """Subtype (ENUM-kind field) should produce a check_enum with member values.""" + assert "check_enum" in generated.content + + def test_country_uses_check_pattern(self, generated: GeneratedModule) -> None: + """Country field (required newtype) produces both check_required and check_pattern.""" + assert "check_pattern" in generated.content + # Bug #1 regression: check_required must not be skipped for required newtype fields. + # With split checks, each descriptor produces its own function; both must appear. + assert "check_required" in generated.content + + def test_region_uses_check_pattern(self, generated: GeneratedModule) -> None: + """Region field produces check_pattern with the region-code label.""" + assert "ISO 3166-2 subdivision code" in generated.content + + +@pytest.mark.parametrize( + "class_name,builder_name,schema_name", + [ + ("DivisionArea", "division_area_checks", "DIVISION_AREA_SCHEMA"), + ("Division", "division_checks", "DIVISION_SCHEMA"), + ("DivisionBoundary", "division_boundary_checks", "DIVISION_BOUNDARY_SCHEMA"), + ("Place", "place_checks", "PLACE_SCHEMA"), + ], +) +class TestModelFeatureGeneration: + @pytest.fixture + def generated(self, class_name: str) -> GeneratedModule: + spec = discover_feature(class_name) + return generate_pyspark_module(spec) + + def test_generates_valid_python( + self, + generated: GeneratedModule, + class_name: str, + builder_name: str, + schema_name: str, + ) -> None: + ast.parse(generated.content) + + def test_has_builder_function( + self, + generated: GeneratedModule, + class_name: str, + builder_name: str, + schema_name: str, + ) -> None: + assert f"def {builder_name}()" in generated.content + + def test_has_schema_constant( + self, + generated: GeneratedModule, + class_name: str, + builder_name: str, + schema_name: str, + ) -> None: + assert schema_name in generated.content + + def test_has_shared_bbox_struct( + self, + generated: GeneratedModule, + class_name: str, + builder_name: str, + schema_name: str, + ) -> None: + assert "BBOX_STRUCT" in generated.content + + +class TestSegmentGeneration: + @pytest.fixture + def generated(self) -> GeneratedModule: + spec = discover_feature("Segment") + return generate_pyspark_module(spec) + + def test_generates_valid_python(self, generated: GeneratedModule) -> None: + ast.parse(generated.content) + + def test_has_builder_and_schema(self, generated: GeneratedModule) -> None: + assert "def segment_checks()" in generated.content + assert "SEGMENT_SCHEMA" in generated.content + + def test_has_shared_bbox_struct(self, generated: GeneratedModule) -> None: + assert "BBOX_STRUCT" in generated.content + + def test_has_variant_conditional_checks(self, generated: GeneratedModule) -> None: + """Segment has subtype-gated fields using runtime values like 'road'.""" + assert "F.when" in generated.content + assert "isin" in generated.content + # Variant values must use the runtime string value, not the enum repr + assert '"road"' in generated.content or "'road'" in generated.content + assert "Subtype.ROAD" not in generated.content + + def test_array_discriminator_outside_lambda( + self, generated: GeneratedModule + ) -> None: + """Top-level discriminator must wrap array_check, not appear inside the lambda.""" + # el["subtype"] must never appear — subtype is a top-level column, not an element field + assert 'el["subtype"]' not in generated.content, ( + 'el["subtype"] found — top-level discriminator placed inside array lambda' + ) + # F.col("subtype") must appear as the discriminator reference + assert 'F.col("subtype")' in generated.content + + +def test_cli_writes_init_modules(tmp_path: Path) -> None: + spec = extract_model(SimpleModel, entry_point="overture.schema.simple:SimpleModel") + out = tmp_path / "src" + test_out = tmp_path / "tests" + _generate_pyspark([spec], out, test_out) + assert (out / "overture" / "schema" / "simple" / "__init__.py").exists() + assert (out / "overture" / "schema" / "simple" / "simple_model.py").exists() + assert (test_out / "overture" / "schema" / "simple" / "__init__.py").exists() + assert ( + test_out / "overture" / "schema" / "simple" / "test_simple_model.py" + ).exists() diff --git a/packages/overture-schema-codegen/tests/test_pyspark_invalid_value.py b/packages/overture-schema-codegen/tests/test_pyspark_invalid_value.py new file mode 100644 index 000000000..9fceb373f --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_pyspark_invalid_value.py @@ -0,0 +1,198 @@ +"""Tests for constraint-violating value generation.""" + +import pytest +from overture.schema.codegen.pyspark.constraint_dispatch import ExpressionDescriptor +from overture.schema.codegen.pyspark.test_data.invalid_value import invalid_value +from overture.schema.system.field_constraint.string import ( + CountryCodeAlpha2Constraint, + JsonPointerConstraint, + NoWhitespaceConstraint, + RegionCodeConstraint, + StrippedConstraint, +) +from overture.schema.system.primitive.geom import GeometryType + + +class TestInvalidValueRequired: + def test_returns_none(self) -> None: + desc = ExpressionDescriptor(function="check_required") + assert invalid_value(desc) is None + + +class TestInvalidValueEnum: + def test_returns_invalid_sentinel(self) -> None: + desc = ExpressionDescriptor(function="check_enum", args=(["a", "b"],)) + assert invalid_value(desc) == "__INVALID__" + + +class TestInvalidValueBounds: + def test_ge(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("ge", 0),)) + assert invalid_value(desc) == -1 + + def test_ge_float(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("ge", 0.0),)) + assert invalid_value(desc) == -1.0 + assert isinstance(invalid_value(desc), float) + + def test_gt(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("gt", 0),)) + assert invalid_value(desc) == 0 + + def test_le(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("le", 100),)) + assert invalid_value(desc) == 101 + + def test_lt(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("lt", 100),)) + assert invalid_value(desc) == 100 + + def test_unknown_bound_raises(self) -> None: + desc = ExpressionDescriptor(function="check_bounds", kwargs=(("unknown", 5),)) + with pytest.raises(ValueError): + invalid_value(desc) + + +class TestInvalidValuePattern: + def test_unknown_constraint_type_raises(self) -> None: + desc = ExpressionDescriptor(function="check_pattern", args=(r"^[A-Z]+$",)) + with pytest.raises(ValueError, match="No invalid value"): + invalid_value(desc) + + def test_no_whitespace_pattern(self) -> None: + desc = ExpressionDescriptor( + function="check_pattern", + args=(r"^\S+$",), + constraint_type=NoWhitespaceConstraint, + ) + assert invalid_value(desc) == "has whitespace" + + +class TestInvalidValueStringTypes: + def test_country_code(self) -> None: + desc = ExpressionDescriptor( + function="check_pattern", + constraint_type=CountryCodeAlpha2Constraint, + ) + assert invalid_value(desc) == "99" + + def test_region_code(self) -> None: + desc = ExpressionDescriptor( + function="check_pattern", + constraint_type=RegionCodeConstraint, + ) + assert invalid_value(desc) == "99-999" + + def test_url_format(self) -> None: + desc = ExpressionDescriptor(function="check_url_format") + assert invalid_value(desc) == "not-a-url" + + def test_url_length(self) -> None: + desc = ExpressionDescriptor(function="check_url_length") + assert invalid_value(desc) == "https://" + "x" * 2076 + + def test_email(self) -> None: + desc = ExpressionDescriptor(function="check_email") + assert invalid_value(desc) == "not-an-email" + + def test_stripped(self) -> None: + desc = ExpressionDescriptor( + function="check_stripped", constraint_type=StrippedConstraint + ) + assert invalid_value(desc) == " has spaces " + + def test_json_pointer(self) -> None: + desc = ExpressionDescriptor( + function="check_json_pointer", constraint_type=JsonPointerConstraint + ) + assert invalid_value(desc) == "no-slash" + + +class TestInvalidValueCollections: + def test_min_length_empty_list(self) -> None: + desc = ExpressionDescriptor(function="check_array_min_length", args=(1,)) + assert invalid_value(desc) == [] + + def test_max_length_oversized(self) -> None: + desc = ExpressionDescriptor(function="check_array_max_length", args=(3,)) + assert invalid_value(desc) == [{}] * 4 + + def test_string_min_length_empty_string(self) -> None: + desc = ExpressionDescriptor(function="check_string_min_length", args=(1,)) + assert invalid_value(desc) == "" + + def test_string_max_length_oversized_string(self) -> None: + desc = ExpressionDescriptor(function="check_string_max_length", args=(3,)) + assert invalid_value(desc) == "x" * 4 + + +class TestInvalidValueLinearRange: + def test_linear_range_length(self) -> None: + desc = ExpressionDescriptor(function="check_linear_range_length") + assert invalid_value(desc) == [0.5] + + def test_linear_range_bounds(self) -> None: + desc = ExpressionDescriptor(function="check_linear_range_bounds") + assert invalid_value(desc) == [1.5, 2.0] + + def test_linear_range_order(self) -> None: + desc = ExpressionDescriptor(function="check_linear_range_order") + assert invalid_value(desc) == [0.8, 0.2] + + +class TestInvalidValueGeometry: + def test_point_not_allowed_picks_point(self) -> None: + # Allowed: polygon only → first candidate (POINT) not in allowed set + desc = ExpressionDescriptor( + function="check_geometry_type", args=(GeometryType.POLYGON,) + ) + assert invalid_value(desc) == "POINT (0 0)" + + def test_point_allowed_picks_linestring(self) -> None: + desc = ExpressionDescriptor( + function="check_geometry_type", + args=(GeometryType.POINT, GeometryType.POLYGON), + ) + assert invalid_value(desc) == "LINESTRING (0 0, 1 1)" + + def test_point_and_linestring_allowed_picks_collection(self) -> None: + desc = ExpressionDescriptor( + function="check_geometry_type", + args=(GeometryType.POINT, GeometryType.LINE_STRING), + ) + assert invalid_value(desc) == "GEOMETRYCOLLECTION EMPTY" + + def test_all_candidates_allowed_raises(self) -> None: + desc = ExpressionDescriptor( + function="check_geometry_type", + args=( + GeometryType.POINT, + GeometryType.LINE_STRING, + GeometryType.GEOMETRY_COLLECTION, + ), + ) + with pytest.raises(ValueError): + invalid_value(desc) + + +class TestInvalidValueRawPattern: + """Raw pydantic `Field(pattern=)` map keys curated in `PATTERN_VALUES`.""" + + def test_curated_license_pattern_returns_invalid(self) -> None: + # Sources.license_priority key pattern, anchor-normalized. + desc = ExpressionDescriptor( + function="check_pattern", args=(r"^[A-Za-z0-9._+\-]+\z",) + ) + assert invalid_value(desc) == "bad license!" + + def test_uncurated_pattern_still_raises(self) -> None: + desc = ExpressionDescriptor(function="check_pattern", args=(r"^xyz\z",)) + with pytest.raises(ValueError, match="check_pattern"): + invalid_value(desc) + + +class TestInvalidValueUnknown: + def test_unknown_function_raises(self) -> None: + desc = ExpressionDescriptor(function="check_something_unknown") + with pytest.raises(ValueError): + invalid_value(desc) diff --git a/packages/overture-schema-codegen/tests/test_pyspark_pipeline.py b/packages/overture-schema-codegen/tests/test_pyspark_pipeline.py new file mode 100644 index 000000000..d84fe7199 --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_pyspark_pipeline.py @@ -0,0 +1,369 @@ +"""Tests for the PySpark generation pipeline.""" + +import ast +from pathlib import PurePosixPath +from typing import Annotated, Literal + +import pytest +from annotated_types import Ge +from codegen_test_support import find_theme +from overture.schema.codegen.extraction.model_extraction import extract_model +from overture.schema.codegen.extraction.specs import ( + ModelSpec, +) +from overture.schema.codegen.pyspark.check_ir import Check +from overture.schema.codegen.pyspark.constraint_dispatch import ExpressionDescriptor +from overture.schema.codegen.pyspark.pipeline import ( + GeneratedModule, + PipelineOutput, + _extract_geometry_types, + generate_pyspark_module, + generate_pyspark_modules, +) +from overture.schema.codegen.spec_discovery import extract_model_spec +from overture.schema.system.field_path import ScalarPath +from overture.schema.system.primitive import GeometryType +from pydantic import BaseModel + + +class SimpleModel(BaseModel): + subtype: Literal["a", "b"] + score: Annotated[float, Ge(0.0)] | None = None + + +class BoundsModel(BaseModel): + value: Annotated[float, Ge(0.0)] + + +class TestGeneratePysparkModule: + @pytest.fixture + def simple_module(self) -> GeneratedModule: + return generate_pyspark_module( + extract_model(SimpleModel, entry_point="overture.schema.simple:SimpleModel") + ) + + def test_returns_generated_module(self, simple_module: GeneratedModule) -> None: + assert isinstance(simple_module, GeneratedModule) + + def test_content_is_nonempty(self, simple_module: GeneratedModule) -> None: + assert simple_module.content + + def test_content_is_valid_python(self, simple_module: GeneratedModule) -> None: + ast.parse(simple_module.content) + + def test_path_uses_snake_case_model_name( + self, simple_module: GeneratedModule + ) -> None: + assert simple_module.path == PurePosixPath( + "overture/schema/simple/simple_model.py" + ) + + def test_path_for_bounds_model(self) -> None: + result = generate_pyspark_module( + extract_model(BoundsModel, entry_point="overture.schema.bounds:BoundsModel") + ) + assert result.path == PurePosixPath("overture/schema/bounds/bounds_model.py") + + def test_content_contains_checks_function( + self, simple_module: GeneratedModule + ) -> None: + assert "simple_model_checks" in simple_module.content + + def test_content_contains_schema_constant( + self, simple_module: GeneratedModule + ) -> None: + assert "SIMPLE_MODEL_SCHEMA" in simple_module.content + + +def _two_specs() -> list[ModelSpec]: + return [ + extract_model(SimpleModel, entry_point="overture.schema.simple:SimpleModel"), + extract_model(BoundsModel, entry_point="overture.schema.bounds:BoundsModel"), + ] + + +def _features(modules: list[GeneratedModule]) -> list[GeneratedModule]: + return [m for m in modules if m.path.name != "__init__.py"] + + +class TestGeneratePysparkModules: + @pytest.fixture + def two_spec_modules(self) -> PipelineOutput: + return generate_pyspark_modules(_two_specs()) + + def test_empty_specs_returns_no_modules(self) -> None: + result = generate_pyspark_modules([]) + assert result.source == [] + assert result.test == [] + + def test_one_module_per_spec(self, two_spec_modules: PipelineOutput) -> None: + assert len(_features(two_spec_modules.source)) == 2 + + def test_paths_unique_per_tree(self, two_spec_modules: PipelineOutput) -> None: + # source and test trees mirror the same dirs; uniqueness is + # only required within each tree, not across them. + for tree in (two_spec_modules.source, two_spec_modules.test): + paths = [m.path for m in tree] + assert len(paths) == len(set(paths)) + + def test_all_content_is_valid_python( + self, two_spec_modules: PipelineOutput + ) -> None: + for mod in (*two_spec_modules.source, *two_spec_modules.test): + ast.parse(mod.content) + + def test_divisions_theme_produces_division_area( + self, all_discovered_models: dict + ) -> None: + """divisions theme should produce a division_area.py module.""" + division_specs: list[ModelSpec] = [] + for key, entry in all_discovered_models.items(): + if find_theme(key.tags) != "divisions": + continue + spec = extract_model_spec(key, entry) + if spec is not None: + division_specs.append(spec) + + results = generate_pyspark_modules(division_specs) + names = {r.path.stem for r in results.source} + assert "division_area" in names + + +class TestTestModuleGeneration: + @pytest.fixture + def all_modules(self) -> PipelineOutput: + return generate_pyspark_modules(_two_specs()) + + def test_generates_test_modules(self, all_modules: PipelineOutput) -> None: + assert len(_features(all_modules.test)) == 2 # one per feature spec + + def test_test_module_paths(self, all_modules: PipelineOutput) -> None: + paths = {m.path.name for m in _features(all_modules.test)} + assert "test_simple_model.py" in paths + assert "test_bounds_model.py" in paths + + def test_test_modules_are_valid_python(self, all_modules: PipelineOutput) -> None: + for mod in all_modules.test: + ast.parse(mod.content) + + def test_test_module_contains_imports(self, all_modules: PipelineOutput) -> None: + for mod in _features(all_modules.test): + assert "_support.harness import" in mod.content + assert "_support.scenarios import" in mod.content + + +def _extract_scenarios_block(content: str) -> str: + """Extract the SCENARIOS list literal from generated test source.""" + start = content.index("SCENARIOS:") + end = content.index("]", start) + 1 + return content[start:end] + + +class TestPerArmTestGeneration: + """Union features with multiple examples produce per-arm test modules.""" + + @pytest.fixture + def segment_modules(self, all_discovered_models: dict) -> PipelineOutput: + specs: list[ModelSpec] = [] + for key, entry in all_discovered_models.items(): + if key.name != "segment": + continue + spec = extract_model_spec(key, entry) + if spec is not None: + specs.append(spec) + return generate_pyspark_modules(specs) + + def test_produces_per_arm_test_files(self, segment_modules: PipelineOutput) -> None: + paths = {m.path.name for m in _features(segment_modules.test)} + assert "test_segment_road.py" in paths + assert "test_segment_rail.py" in paths + + def test_no_monolithic_test_file(self, segment_modules: PipelineOutput) -> None: + """When per-arm tests exist, no undifferentiated test_segment.py.""" + paths = {m.path.name for m in _features(segment_modules.test)} + assert "test_segment.py" not in paths + + def test_per_arm_modules_are_valid_python( + self, segment_modules: PipelineOutput + ) -> None: + for mod in segment_modules.test: + ast.parse(mod.content) + + def test_road_module_has_road_checks(self, segment_modules: PipelineOutput) -> None: + road = next( + m for m in segment_modules.test if m.path.name == "test_segment_road.py" + ) + assert "road_surface" in road.content + + def test_rail_module_has_rail_checks(self, segment_modules: PipelineOutput) -> None: + rail = next( + m for m in segment_modules.test if m.path.name == "test_segment_rail.py" + ) + assert "rail_flags" in rail.content + + def test_road_module_no_rail_field_scenarios( + self, segment_modules: PipelineOutput + ) -> None: + road = next( + m for m in segment_modules.test if m.path.name == "test_segment_road.py" + ) + scenarios = _extract_scenarios_block(road.content) + assert "rail_flags[].values" not in scenarios + + def test_rail_module_no_road_field_scenarios( + self, segment_modules: PipelineOutput + ) -> None: + rail = next( + m for m in segment_modules.test if m.path.name == "test_segment_rail.py" + ) + scenarios = _extract_scenarios_block(rail.content) + assert "road_surface" not in scenarios + + def test_non_union_still_gets_single_test(self) -> None: + """Non-union features produce a single test module (unchanged).""" + modules = generate_pyspark_modules( + [ + extract_model( + SimpleModel, entry_point="overture.schema.simple:SimpleModel" + ) + ] + ) + tests = _features(modules.test) + assert len(tests) == 1 + assert tests[0].path.name == "test_simple_model.py" + + +class TestNestedSourcePaths: + def test_module_path_mirrors_entry_point(self) -> None: + spec = extract_model( + SimpleModel, entry_point="overture.schema.simple:SimpleModel" + ) + modules = generate_pyspark_modules([spec]) + features = _features(modules.source) + assert len(features) == 1 + assert features[0].path == PurePosixPath( + "overture/schema/simple/simple_model.py" + ) + + def test_two_packages_no_collision(self) -> None: + a = extract_model(SimpleModel, entry_point="overture.schema.places:Place") + b = extract_model(SimpleModel, entry_point="annex.schema.places:Place") + modules = generate_pyspark_modules([a, b]) + paths = {m.path for m in _features(modules.source)} + assert PurePosixPath("overture/schema/places/place.py") in paths + assert PurePosixPath("annex/schema/places/place.py") in paths + + +_EXPECTED_INIT_PATHS = { + PurePosixPath("__init__.py"), + PurePosixPath("overture/__init__.py"), + PurePosixPath("overture/schema/__init__.py"), + PurePosixPath("overture/schema/simple/__init__.py"), +} + + +def _init_paths(modules: list[GeneratedModule]) -> set[PurePosixPath]: + return {m.path for m in modules if m.path.name == "__init__.py"} + + +class TestInitModuleEmission: + def test_intermediate_dirs_get_init_modules(self) -> None: + spec = extract_model( + SimpleModel, entry_point="overture.schema.simple:SimpleModel" + ) + modules = generate_pyspark_modules([spec]) + assert _init_paths(modules.source) == _EXPECTED_INIT_PATHS + + def test_init_modules_are_empty(self) -> None: + spec = extract_model( + SimpleModel, entry_point="overture.schema.simple:SimpleModel" + ) + modules = generate_pyspark_modules([spec]) + init = next(m for m in modules.source if m.path.name == "__init__.py") + assert init.content == "" + + def test_shared_dirs_emitted_once(self) -> None: + a = extract_model(SimpleModel, entry_point="overture.schema.simple:SimpleModel") + b = extract_model(BoundsModel, entry_point="overture.schema.simple:BoundsModel") + modules = generate_pyspark_modules([a, b]) + init_paths = [m.path for m in modules.source if m.path.name == "__init__.py"] + assert len(init_paths) == len(set(init_paths)) + + +class TestNoRegistryEmitted: + def test_registry_module_is_no_longer_generated(self) -> None: + # The runtime builds the registry via entry-point discovery; codegen + # must not emit `_registry.py`. + spec = extract_model( + SimpleModel, entry_point="overture.schema.simple:SimpleModel" + ) + modules = generate_pyspark_modules([spec]) + for tree in (modules.source, modules.test): + assert all(m.path.name != "_registry.py" for m in tree) + + +class TestNestedTestPaths: + def test_test_module_path_mirrors_source(self) -> None: + spec = extract_model( + SimpleModel, entry_point="overture.schema.simple:SimpleModel" + ) + modules = generate_pyspark_modules([spec]) + tests = _features(modules.test) + assert len(tests) == 1 + assert tests[0].path == PurePosixPath( + "overture/schema/simple/test_simple_model.py" + ) + + def test_test_module_imports_nested_expression(self) -> None: + spec = extract_model( + SimpleModel, entry_point="overture.schema.simple:SimpleModel" + ) + modules = generate_pyspark_modules([spec]) + test_mod = next(iter(_features(modules.test))) + assert ( + "from overture.schema.pyspark.expressions.generated.overture.schema.simple.simple_model import" + in test_mod.content + ) + + def test_test_dirs_get_init_modules(self) -> None: + spec = extract_model( + SimpleModel, entry_point="overture.schema.simple:SimpleModel" + ) + modules = generate_pyspark_modules([spec]) + # Source-tree init modules already covered in TestInitModuleEmission. + # The test tree must mirror the same package layout. + assert _init_paths(modules.test) == _EXPECTED_INIT_PATHS + + +class TestExtractGeometryTypes: + """`_extract_geometry_types` aggregates across descriptors and checks.""" + + def test_aggregates_across_descriptors(self) -> None: + checks = [ + Check( + descriptors=( + ExpressionDescriptor( + function="check_geometry_type", + args=(GeometryType.POINT,), + ), + ), + target=ScalarPath(), + ), + Check( + descriptors=( + ExpressionDescriptor( + function="check_geometry_type", + args=(GeometryType.POLYGON, GeometryType.LINE_STRING), + ), + ), + target=ScalarPath(), + ), + ] + assert _extract_geometry_types(checks) == ( + GeometryType.LINE_STRING, + GeometryType.POINT, + GeometryType.POLYGON, + ) + + def test_returns_empty_when_absent(self) -> None: + assert _extract_geometry_types([]) == () diff --git a/packages/overture-schema-codegen/tests/test_pyspark_renderer.py b/packages/overture-schema-codegen/tests/test_pyspark_renderer.py new file mode 100644 index 000000000..27f0dd396 --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_pyspark_renderer.py @@ -0,0 +1,1832 @@ +"""Tests for pyspark feature module renderer.""" + +import ast +import re +from enum import Enum +from typing import Annotated, Literal, Union + +import pytest +from annotated_types import Ge, MinLen +from codegen_test_support import ( + LiteralSubtypeModel, + RadioModel, + RequireAnyModel, + RequireAnyTrueModel, + TripleNestedArrayModel, + discover_feature, + flat_specs_from_discovery, + spec_for_model, +) +from overture.schema.codegen.extraction.specs import ModelSpec +from overture.schema.codegen.pyspark._render_common import ( + field_check_rows, + jinja_env, + model_check_rows, + schema_const_name, +) +from overture.schema.codegen.pyspark.check_builder import build_checks +from overture.schema.codegen.pyspark.check_ir import ( + Check, + ColumnGuard, + ElementGuard, + ModelCheck, +) +from overture.schema.codegen.pyspark.constraint_dispatch import ( + ExpressionDescriptor, + FieldEq, + ForbidIf, + MinFieldsSet, + RadioGroup, + RequireAnyOf, + RequireIf, + require_field_eq, +) +from overture.schema.codegen.pyspark.renderer import ( + _render_check_function_context, + _render_model_constraint_function_context, + render_model_module, +) +from overture.schema.codegen.pyspark.schema_builder import SchemaField, build_schema +from overture.schema.system.field_path import ( + ScalarPath, + parse, +) +from overture.schema.system.model_constraint import ( + FieldEqCondition, + Not, + forbid_if, + require_any_of, + require_if, +) +from overture.schema.system.primitive import ( + Geometry, + GeometryType, + GeometryTypeConstraint, + int32, +) +from overture.schema.system.string import CountryCodeAlpha2 +from pydantic import BaseModel, HttpUrl +from pydantic.fields import FieldInfo + +_path = parse + + +class TestCheckIRReadColumns: + """IR-derived `read_columns` on `Check` and `ModelCheck`. + + Each variant enumerates top-level row columns from the IR structure + directly -- no regex over rendered source. `ColumnGuard` discriminators + are included (they produce `F.col(...)` at the row level); `ElementGuard` + discriminators are not (they reference `el[...]`, an element-relative + accessor). + """ + + def test_scalar_field_read_columns(self) -> None: + check = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("speed"), + ) + assert check.read_columns == frozenset({"speed"}) + + def test_struct_dotted_scalar_strips_to_top_level(self) -> None: + check = Check( + descriptors=(ExpressionDescriptor(function="check_bounds"),), + target=_path("bbox.xmin"), + ) + assert check.read_columns == frozenset({"bbox"}) + + def test_array_field_read_columns(self) -> None: + check = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("sources[]"), + ) + assert check.read_columns == frozenset({"sources"}) + + def test_dotted_array_strips_to_top_level(self) -> None: + check = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("names.rules[]"), + ) + assert check.read_columns == frozenset({"names"}) + + def test_map_field_read_columns(self) -> None: + check = Check( + descriptors=(ExpressionDescriptor(function="check_stripped"),), + target=_path("names.common{value}"), + ) + assert check.read_columns == frozenset({"names"}) + + def test_column_guard_discriminator_included(self) -> None: + check = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("class"), + guards=(ColumnGuard(discriminator="subtype", values=("road",)),), + ) + assert check.read_columns == frozenset({"class", "subtype"}) + + def test_element_guard_discriminator_excluded(self) -> None: + # ElementGuard discriminators reference `el["subtype"]`, not F.col -- + # they are element-relative and do not constitute a top-level row read. + check = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("items[].value"), + guards=(ElementGuard(discriminator="subtype", values=("road",)),), + ) + assert check.read_columns == frozenset({"items"}) + + def test_model_check_require_any_of_reads_all_field_names(self) -> None: + check = ModelCheck( + descriptor=RequireAnyOf(field_names=("x", "y", "z")), + ) + assert check.read_columns == frozenset({"x", "y", "z"}) + + def test_model_check_require_if_includes_condition_field(self) -> None: + check = ModelCheck( + descriptor=RequireIf( + field_names=("admin_level",), + condition=FieldEqCondition("subtype", "county"), + ), + ) + assert check.read_columns == frozenset({"admin_level", "subtype"}) + + def test_model_check_negated_condition_field_included(self) -> None: + check = ModelCheck( + descriptor=RequireIf( + field_names=("admin_level",), + condition=Not(FieldEqCondition("subtype", "county")), + ), + ) + assert check.read_columns == frozenset({"admin_level", "subtype"}) + + def test_model_check_array_target_reads_only_container_column(self) -> None: + # When the constrained model is inside an array, field references use + # element-relative el["x"] accessors (not F.col("x")), so only the outer + # array column is a top-level row read. + check = ModelCheck( + descriptor=RequireAnyOf(field_names=("x", "y")), + target=_path("items[]"), + ) + assert check.read_columns == frozenset({"items"}) + + # IMPORTANT 1 — descriptor gate on scalar vs array target + def test_scalar_target_gate_column_included(self) -> None: + # A descriptor gate on a scalar target renders as F.col("{gate}").isNotNull(), + # a row-level read; the gate's top-level column must appear in read_columns. + check = Check( + descriptors=( + ExpressionDescriptor(function="check_required", gate=_path("parent")), + ), + target=_path("parent.value"), + ) + assert check.read_columns == frozenset({"parent"}) + + def test_array_target_gate_column_excluded(self) -> None: + # A descriptor gate on an array target is applied element-relatively via + # element_relative_gate (el[...]), not as F.col -- excluded from read_columns. + check = Check( + descriptors=( + ExpressionDescriptor(function="check_required", gate=_path("items[]")), + ), + target=_path("items[].value"), + ) + assert check.read_columns == frozenset({"items"}) + + # IMPORTANT 2 — RequireIf condition field exclusion on array target + def test_model_check_require_if_array_target_excludes_condition_field(self) -> None: + # On an array target, the condition is el["cond"] (element-relative), not + # F.col("cond"); only the outer array column is a row-level read. + check = ModelCheck( + descriptor=RequireIf( + field_names=("x",), + condition=FieldEqCondition("cond", "v"), + ), + target=_path("items[]"), + ) + assert check.read_columns == frozenset({"items"}) + + def test_model_check_forbid_if_array_target_excludes_condition_field(self) -> None: + check = ModelCheck( + descriptor=ForbidIf( + field_names=("x",), + condition=FieldEqCondition("cond", "v"), + field_shapes=(), + ), + target=_path("items[]"), + ) + assert check.read_columns == frozenset({"items"}) + + # MINOR 3 — RadioGroup and MinFieldsSet share the RequireAnyOf match arm + @pytest.mark.parametrize( + "descriptor", + [ + RequireAnyOf(field_names=("a", "b")), + RadioGroup(field_names=("a", "b")), + MinFieldsSet(field_names=("a", "b"), count=1), + ], + ) + def test_model_check_row_root_field_names_in_read_columns( + self, descriptor: RequireAnyOf | RadioGroup | MinFieldsSet + ) -> None: + # All three variants carry field_names rendered as F.col(...) at the row root. + check = ModelCheck(descriptor=descriptor) + assert check.read_columns == frozenset({"a", "b"}) + + # MINOR 4 — ModelCheck on a MapPath target + def test_model_check_map_target_reads_only_map_column(self) -> None: + # A dict[K, Model] value-model constraint targets a MapPath; field references + # use the projected element variable (v["field"]), not F.col. Only the map + # column itself is a row-level read. + check = ModelCheck( + descriptor=RequireAnyOf(field_names=("label", "value")), + target=_path("names.common{value}"), + ) + assert check.read_columns == frozenset({"names"}) + + +# Resurrected as a TEST ORACLE: the regex `read_columns` derivation deleted from +# renderer.py in this refactor. It recognized every top-level column form the +# renderer emits -- `F.col`, the outer `array_check`/`nested_array_check`, and +# `map_keys_check`/`map_values_check` -- by reading the rendered source directly. +_RENDERED_COLUMN_READ = re.compile( + r'(?:F\.col|(?:nested_)?array_check|map_(?:keys|values)_check)\("([^"]+)"' +) + + +def _columns_in_rendered_expr(expr: str) -> frozenset[str]: + """Top-level columns a rendered check expression dereferences (regex oracle).""" + return frozenset( + m.group(1).split(".", 1)[0] for m in _RENDERED_COLUMN_READ.finditer(expr) + ) + + +def _read_columns_mismatches(spec: ModelSpec) -> list[str]: + """Mismatches between IR `read_columns` and the rendered source, for one spec.""" + field_checks, model_checks = build_checks(spec) + mismatches: list[str] = [] + for check in field_checks: + for row in field_check_rows([check]): + expr = str(_render_check_function_context(row)["expr"]) + rendered = _columns_in_rendered_expr(expr) + expected = row.check.read_columns + if rendered != expected: + mismatches.append( + f"{spec.name}.{row.label}: rendered={sorted(rendered)} " + f"read_columns={sorted(expected)} expr={expr}" + ) + for model_row in model_check_rows(model_checks): + expr = str(_render_model_constraint_function_context(model_row)["expr"]) + rendered = _columns_in_rendered_expr(expr) + expected = model_row.check.read_columns + if rendered != expected: + mismatches.append( + f"{spec.name}.{model_row.label} (model): rendered={sorted(rendered)} " + f"read_columns={sorted(expected)} expr={expr}" + ) + return mismatches + + +class TestReadColumnsMatchRenderedSource: + """IR-derived `read_columns` equals what the rendered expression dereferences. + + `read_columns` moved from a regex over rendered source to an IR-structural + derivation, so the two are now independent code paths that must agree: + `validate_model` drops a check only when none of its `read_columns` are + present in the input, so a column the rendered `F.col(...)` reads but + `read_columns` omits would reach Spark unresolved when that column is absent. + Neither the unit tests nor the regeneration diff catch such a desync (both + sides derive from the same code). This oracle re-derives the columns from + rendered source -- ground truth -- and asserts equality across every check + the real schemas produce, so a future renderer change that emits a new + column form without updating `read_columns` fails here. + """ + + def test_real_models_read_columns_match_rendered_source(self) -> None: + specs: list[ModelSpec] = list(flat_specs_from_discovery()) + specs.append(discover_feature("Segment")) + mismatches: list[str] = [] + for spec in specs: + mismatches.extend(_read_columns_mismatches(spec)) + assert not mismatches, "read_columns desync:\n" + "\n".join(mismatches) + + +class TestRequireFieldEq: + """`require_field_eq` is the strict, raising companion to `parse_field_eq`.""" + + def test_unwraps_field_eq(self) -> None: + assert require_field_eq(FieldEqCondition("subtype", "county")) == FieldEq( + "subtype", "county", False + ) + + def test_unwraps_negated_field_eq(self) -> None: + condition = Not(FieldEqCondition("subtype", "county")) + assert require_field_eq(condition) == FieldEq("subtype", "county", True) + + def test_raises_on_other_condition(self) -> None: + # A condition `parse_field_eq` cannot unwrap (nested negation) names its + # type in the error so a new Condition subtype fails loudly in one place. + condition = Not(Not(FieldEqCondition("subtype", "county"))) + with pytest.raises(TypeError, match="Unhandled condition type: Not"): + require_field_eq(condition) + + +class TestSchemaConstName: + def test_uppercases_model_name(self) -> None: + assert schema_const_name("address") == "ADDRESS_SCHEMA" + + def test_already_uppercase(self) -> None: + assert schema_const_name("BUILDING") == "BUILDING_SCHEMA" + + def test_mixed_case(self) -> None: + assert schema_const_name("myFeature") == "MYFEATURE_SCHEMA" + + +class BoundsModel(BaseModel): + score: Annotated[float, Ge(0.0)] + + +# int32 is a non-float primitive; bounds on it must not emit the NaN guard. +class IntBoundsModel(BaseModel): + count: Annotated[int32, Ge(0)] + + +class ArrayModel(BaseModel): + tags: Annotated[list[str], MinLen(1)] + + +class InnerModel(BaseModel): + value: str + + +class NestedArrayModel(BaseModel): + items: list[InnerModel] | None = None + + +# list[Annotated[float, Ge(0.0)]] produces ARRAY-shape nodes because +# check_bounds is an element-level function (not in _COLUMN_LEVEL_FUNCTIONS). +class FloatListModel(BaseModel): + scores: list[Annotated[float, Ge(0.0)]] | None = None + + +class MapValueLeaf(BaseModel): + label: Annotated[str, MinLen(1)] + + +# dict[K, Model] value model with a constrained field -- the field check +# renders inside a map_values_check lambda navigating into the value struct. +class MapValueFieldModel(BaseModel): + items: dict[str, MapValueLeaf] + + +@require_any_of("foo", "bar") +class MapValueAnyOf(BaseModel): + foo: int | None = None + bar: str | None = None + + +# dict[K, Model] value model with a model-level constraint -- the model check +# renders inside a map_values_check lambda. +class MapValueConstraintModel(BaseModel): + subs: dict[str, MapValueAnyOf] + + +@require_if(["admin_level"], FieldEqCondition("subtype", "country")) +class LeafRequireIf(BaseModel): + subtype: str + admin_level: int | None = None + + +# The require_if model sits one struct level below the container element, so +# both container types reach it at a non-empty leaf (`...inner`). The target +# AND condition field refs must both navigate that leaf. +class LeafRequireIfOuter(BaseModel): + inner: LeafRequireIf + + +class MapValueRequireIfModel(BaseModel): + subs: dict[str, LeafRequireIfOuter] + + +class ArrayValueRequireIfModel(BaseModel): + rows: list[LeafRequireIfOuter] + + +@forbid_if(["extra"], FieldEqCondition("kind", "basic")) +class MapValueForbidIf(BaseModel): + kind: str + extra: str | None = None + + +class MapValueForbidIfModel(BaseModel): + subs: dict[str, MapValueForbidIf] + + +def _render(model_cls: type[BaseModel], name: str = "simple") -> str: + spec = spec_for_model(model_cls) + field_checks, model_checks = build_checks(spec) + schema_fields = build_schema(spec) + return render_model_module(name, field_checks, model_checks, schema_fields) + + +def _render_check_function_string(ctx: dict[str, object]) -> str: + """Render a single check function context to source via the Jinja macro.""" + template = jinja_env().get_template("_check_function.py.jinja2") + return str(template.module.check_function(c=ctx)) # type: ignore[attr-defined] + + +def _render_check_function(check: Check, descriptor_idx: int = 0) -> str: + """Render a per-field check function source from a Check.""" + row = field_check_rows([check])[descriptor_idx] + ctx = _render_check_function_context(row) + return _render_check_function_string(ctx) + + +def _render_node(check: Check) -> str: + """Render a single Check to its function source.""" + return _render_check_function(check, descriptor_idx=0) + + +def _render_model_node(check: ModelCheck) -> str: + """Render a single ModelCheck to its function source.""" + ctx = _render_model_constraint_function_context(model_check_rows([check])[0]) + return _render_check_function_string(ctx) + + +@pytest.fixture(scope="module") +def literal_subtype_source() -> str: + """Rendered `LiteralSubtypeModel` source (default `simple` feature name). + + Module-scoped so the extraction+render cost is paid once for all + consumers in this file. + """ + return _render(LiteralSubtypeModel) + + +class TestParseable: + def test_renders_parseable_python(self, literal_subtype_source: str) -> None: + ast.parse(literal_subtype_source) + + def test_bounds_model_parseable(self) -> None: + source = _render(BoundsModel) + ast.parse(source) + + def test_array_model_parseable(self) -> None: + source = _render(ArrayModel) + ast.parse(source) + + def test_nested_array_model_parseable(self) -> None: + source = _render(NestedArrayModel) + ast.parse(source) + + def test_radio_model_parseable(self) -> None: + source = _render(RadioModel, "radio") + ast.parse(source) + + def test_require_any_model_parseable(self) -> None: + source = _render(RequireAnyModel, "require_any") + ast.parse(source) + + def test_depth_3_renders_valid_python(self) -> None: + source = _render(TripleNestedArrayModel, "triple") + ast.parse(source) + assert "nested_array_check(" in source + + +class TestBoundsNanGuardRendering: + """The NaN guard flag is emitted only for non-float bound columns.""" + + def test_float_bound_omits_check_nan(self) -> None: + """Float-typed bounds produce a check_bounds call without check_nan.""" + source = _render(BoundsModel) + start = source.index("check_bounds(") + call = source[start : source.index(")", start) + 1] + assert "check_nan" not in call + + def test_integer_bound_emits_check_nan_false(self) -> None: + """Integer-typed bounds include check_nan=False to skip the dead guard.""" + source = _render(IntBoundsModel) + assert "check_nan=False" in source + + +class TestBuilderFunction: + def test_contains_builder_function(self, literal_subtype_source: str) -> None: + assert "def simple_checks()" in literal_subtype_source + + def test_builder_returns_list_check(self, literal_subtype_source: str) -> None: + assert "list[Check]" in literal_subtype_source + + def test_builder_name_uses_model_name(self) -> None: + source = _render(LiteralSubtypeModel, "my_model") + assert "def my_model_checks()" in source + + +class TestSchemaConstant: + def test_contains_schema_constant(self, literal_subtype_source: str) -> None: + assert "SIMPLE_SCHEMA" in literal_subtype_source + + def test_schema_constant_name_uppercased(self) -> None: + source = _render(LiteralSubtypeModel, "my_feature") + assert "MY_FEATURE_SCHEMA" in source + + def test_contains_struct_type(self, literal_subtype_source: str) -> None: + assert "StructType" in literal_subtype_source + + def test_contains_struct_field(self, literal_subtype_source: str) -> None: + assert "StructField" in literal_subtype_source + + def test_shared_struct_ref_emits_struct_field(self) -> None: + """Shared struct refs (BBOX_STRUCT) render as the type of a StructField.""" + schema_fields = [SchemaField(name="bbox", type_expr="BBOX_STRUCT")] + source = render_model_module("simple", [], [], schema_fields) + assert 'StructField("bbox", BBOX_STRUCT, True)' in source + + +class TestGeometryTypes: + """`GEOMETRY_TYPES` constant emission for runtime discovery.""" + + def test_omitted_when_empty(self, literal_subtype_source: str) -> None: + assert "GEOMETRY_TYPES" not in literal_subtype_source + + def test_emitted_when_provided(self) -> None: + spec = spec_for_model(LiteralSubtypeModel) + field_nodes, model_nodes = build_checks(spec) + schema_fields = build_schema(spec) + source = render_model_module( + "simple", + field_nodes, + model_nodes, + schema_fields, + geometry_types=(GeometryType.POINT,), + ) + assert ( + "GEOMETRY_TYPES: tuple[GeometryType, ...] = (GeometryType.POINT,)" in source + ) + + def test_geometry_type_imported_when_only_constant_needs_it(self) -> None: + # LiteralSubtypeModel has no check_geometry_type constraint, so the + # import is only required because GEOMETRY_TYPES references it. + spec = spec_for_model(LiteralSubtypeModel) + field_nodes, model_nodes = build_checks(spec) + schema_fields = build_schema(spec) + source = render_model_module( + "simple", + field_nodes, + model_nodes, + schema_fields, + geometry_types=(GeometryType.POINT,), + ) + assert "from overture.schema.system.primitive import GeometryType" in source + + +class TestImports: + def test_imports_pyspark_functions(self, literal_subtype_source: str) -> None: + assert "from pyspark.sql import functions as F" in literal_subtype_source + + def test_imports_check_classes(self, literal_subtype_source: str) -> None: + assert ( + "from overture.schema.pyspark.check import Check, CheckShape" + in literal_subtype_source + ) + + def test_imports_constraint_expressions(self, literal_subtype_source: str) -> None: + assert ( + "from overture.schema.pyspark.expressions.constraint_expressions import" + in literal_subtype_source + ) + + def test_imports_schema_types(self, literal_subtype_source: str) -> None: + # StructType and StructField must appear in the import section (before first def) + first_def = literal_subtype_source.index("\ndef ") + import_section = literal_subtype_source[:first_def] + assert "pyspark.sql.types" in import_section + assert "StructType" in import_section + assert "StructField" in import_section + + def test_imports_array_check_when_needed(self) -> None: + source = _render(FloatListModel, "float_list") + assert "array_check" in source + + def test_no_unused_column_patterns_import_for_simple( + self, literal_subtype_source: str + ) -> None: + # LiteralSubtypeModel has no array fields -- column_patterns import not needed + assert "column_patterns" not in literal_subtype_source + + +class TestPerFieldFunctions: + def test_per_field_function_exists(self, literal_subtype_source: str) -> None: + # With split checks, compound fields produce suffixed names + assert ( + "_subtype_required_check" in literal_subtype_source + or "_subtype_enum_check" in literal_subtype_source + ) + + def test_check_has_name_field(self, literal_subtype_source: str) -> None: + """Rendered Check includes name= derived from constraint function.""" + assert "name='required'" in literal_subtype_source + assert "name='enum'" in literal_subtype_source + + def test_no_field_in_check_calls(self, literal_subtype_source: str) -> None: + """check_* calls should not include field string as second arg.""" + # Match pattern: check_xxx(F.col("yyy"), "yyy", ...) — field as 2nd arg + field_arg_pattern = re.compile(r'check_\w+\(F\.col\("[^"]+"\),\s*"[^"]+"') + assert not field_arg_pattern.search(literal_subtype_source) + + def test_scalar_single_descriptor_no_coalesce(self) -> None: + class OptionalBounds(BaseModel): + value: Annotated[float, Ge(0.0)] | None = None + + source = _render(OptionalBounds, "opt") + assert "check_bounds" in source + assert "F.coalesce" not in source + + def test_scalar_multi_descriptor_produces_separate_checks( + self, literal_subtype_source: str + ) -> None: + """SimpleModel.subtype has check_required + check_enum -> two separate functions.""" + assert "F.coalesce" not in literal_subtype_source + assert "name='required'" in literal_subtype_source + assert "name='enum'" in literal_subtype_source + + def test_compound_checks_split(self, literal_subtype_source: str) -> None: + """A field with required + enum produces two Check functions, not one coalesced.""" + assert "F.coalesce" not in literal_subtype_source + + def test_array_shape_uses_array_check(self) -> None: + source = _render(FloatListModel, "float_list") + assert "array_check" in source + + def test_field_function_name_sanitized(self) -> None: + # nested field like "items[].value" -> _items_value_check + source = _render(NestedArrayModel) + assert "_items_value_check" in source + + def test_builder_collects_all_checks(self, literal_subtype_source: str) -> None: + # With split checks, both descriptors appear in the builder + assert "_subtype_required_check()" in literal_subtype_source + assert "_subtype_enum_check()" in literal_subtype_source + + +class TestModelConstraintFunctions: + def test_radio_group_check_rendered(self) -> None: + source = _render(RadioModel, "radio") + assert "check_radio_group" in source + + def test_require_any_of_rendered(self) -> None: + source = _render(RequireAnyModel, "require_any") + assert "check_require_any_of" in source + + def test_require_any_true_rendered(self) -> None: + source = _render(RequireAnyTrueModel, "require_any_true") + assert "check_require_any_true" in source + + def test_require_any_true_lowers_conditions_to_value_equality(self) -> None: + """Each FieldEqCondition(field, True) lowers to a boolean Column. + + The bool value renders as `F.lit(True)` (not the `True` keyword), + so ruff's E712 does not rewrite the comparison away. + """ + source = _render(RequireAnyTrueModel, "require_any_true") + assert 'F.col("is_land") == F.lit(True)' in source + assert 'F.col("is_territorial") == F.lit(True)' in source + + def test_require_any_true_reads_condition_columns(self) -> None: + source = _render(RequireAnyTrueModel, "require_any_true") + assert "read_columns=frozenset({'is_land', 'is_territorial'})" in source + + def test_radio_group_no_context_arg(self) -> None: + """check_radio_group must not receive a context string argument.""" + source = _render(RadioModel, "radio") + # Context arg was the model name, e.g. "RadioModel" — must not appear + assert "'RadioModel'" not in source + + def test_require_any_of_no_context_arg(self) -> None: + """check_require_any_of must not receive a context string argument.""" + source = _render(RequireAnyModel, "require_any") + assert "'RequireAnyModel'" not in source + + def test_require_any_of_emits_read_columns(self) -> None: + # A model check reads several columns directly; the runtime drops the + # check when any of them is skipped or structurally absent. + source = _render(RequireAnyModel, "require_any") + assert "read_columns=frozenset({'x', 'y'})" in source + + def test_field_check_emits_read_columns(self) -> None: + # Every check declares the columns it reads, field checks included -- + # there is no separate root_field/referenced_fields split. + source = _render(BoundsModel) + assert "read_columns=frozenset({'score'})" in source + assert "root_field" not in source + assert "referenced_fields" not in source + + def test_require_if_read_columns_include_condition(self) -> None: + # require_if reads its target column and the column its condition + # branches on; both must be carried so skipping either drops the check. + source = _render(RequireIfEnumModel, "require_if_enum") + assert "read_columns=frozenset({'admin_level', 'subtype'})" in source + + def test_model_constraint_imports_function(self) -> None: + source = _render(RadioModel, "radio") + assert "check_radio_group" in source + # imported from constraint_expressions + assert ( + "from overture.schema.pyspark.expressions.constraint_expressions import" + in source + ) + + def test_model_constraint_included_in_builder(self) -> None: + source = _render(RadioModel, "radio") + # some check function for radio_group should appear in builder return + lines = source.splitlines() + builder_lines = [] + in_builder = False + for line in lines: + if "def radio_checks()" in line: + in_builder = True + if in_builder: + builder_lines.append(line) + builder_src = "\n".join(builder_lines) + assert "check" in builder_src.lower() + + +class TestEnumConstants: + def test_enum_values_appear_as_list(self, literal_subtype_source: str) -> None: + for value in ("a", "b", "c"): + assert f"'{value}'" in literal_subtype_source + + def test_check_enum_called_with_values(self, literal_subtype_source: str) -> None: + assert "check_enum" in literal_subtype_source + + +class GeomModel(BaseModel): + geometry: Annotated[ + Geometry, + GeometryTypeConstraint(GeometryType.POLYGON, GeometryType.MULTI_POLYGON), + ] + + +class TestGeometryTypeRendering: + def test_geometry_type_renders_valid_python(self) -> None: + source = _render(GeomModel, "geom") + ast.parse(source) + + def test_geometry_type_uses_qualified_name(self) -> None: + source = _render(GeomModel, "geom") + assert "GeometryType.POLYGON" in source + assert "GeometryType.MULTI_POLYGON" in source + + def test_geometry_type_import_present(self) -> None: + source = _render(GeomModel, "geom") + assert "from overture.schema.system.primitive import GeometryType" in source + + def test_no_geometry_type_import_without_geometry_field( + self, literal_subtype_source: str + ) -> None: + assert "GeometryType" not in literal_subtype_source + + +class _DeepInner(BaseModel): + field: str + + +class _ArrayElementWithNestedStruct(BaseModel): + nested: _DeepInner + + +class DeepNestedArrayModel(BaseModel): + items: list[_ArrayElementWithNestedStruct] + + +class _ArrayElementWithList(BaseModel): + countries: list[CountryCodeAlpha2] + + +class ListInArrayModel(BaseModel): + items: list[_ArrayElementWithList] + + +class _ArrayElementWithNewtype(BaseModel): + country: Annotated[str, Ge(0)] # stand-in for a constrained field + + +class TestArrayElementSubfieldRendering: + """Scalar sub-fields of array elements render as array_check with el[...] accessors.""" + + def test_scalar_subfield_uses_array_check(self) -> None: + source = _render(NestedArrayModel, "nested") + assert "array_check(" in source + + def test_scalar_subfield_uses_element_accessor(self) -> None: + source = _render(NestedArrayModel, "nested") + assert 'el["value"]' in source + + def test_scalar_subfield_no_f_col_with_brackets(self) -> None: + source = _render(NestedArrayModel, "nested") + assert 'F.col("items[].value")' not in source + + def test_nested_struct_subfield_chained_brackets(self) -> None: + source = _render(DeepNestedArrayModel, "deep") + assert 'el["nested"]["field"]' in source + + def test_nested_struct_subfield_no_dot_in_brackets(self) -> None: + source = _render(DeepNestedArrayModel, "deep") + assert 'el["nested.field"]' not in source + + def test_list_subfield_uses_nested_array_check(self) -> None: + source = _render(ListInArrayModel, "list_in_array") + assert "nested_array_check(" in source + + def test_list_subfield_has_inner_array_check(self) -> None: + source = _render(ListInArrayModel, "list_in_array") + # nested_array_check outer + array_check inner + assert "nested_array_check(" in source + assert "array_check(" in source + + def test_list_subfield_parseable(self) -> None: + source = _render(ListInArrayModel, "list_in_array") + ast.parse(source) + + def test_deep_nested_parseable(self) -> None: + source = _render(DeepNestedArrayModel, "deep") + ast.parse(source) + + +class TestNoFunctionNameCollisions: + def test_list_field_produces_unique_function_names(self) -> None: + source = _render(ArrayModel, "arr") + # Each "def _" function name should appear exactly once + func_defs = re.findall(r"^def (_\w+_check)\(", source, re.MULTILINE) + assert len(func_defs) == len(set(func_defs)), ( + f"Duplicate function names: {func_defs}" + ) + + def test_list_field_renders_parseable(self) -> None: + source = _render(ArrayModel, "arr") + ast.parse(source) + + +class PlaceSubtype(str): + COUNTRY = "country" + REGION = "region" + + def __new__(cls, value: str) -> "PlaceSubtype": + return str.__new__(cls, value) + + +class _SubtypeEnum(str, Enum): + COUNTRY = "country" + REGION = "region" + + +@require_if(["admin_level"], FieldEqCondition("subtype", _SubtypeEnum.COUNTRY)) +class RequireIfEnumModel(BaseModel): + subtype: str + admin_level: int | None = None + + +class TestModelConstraintNoRedundantArgs: + """Model constraints must not embed context or target_name strings.""" + + def test_require_if_no_target_name_arg(self) -> None: + """check_require_if must not pass the field name as a string arg.""" + source = _render(RequireIfEnumModel, "require_if_enum") + # Was: check_require_if(F.col("admin_level"), "admin_level", condition, desc) + # Now: check_require_if(F.col("admin_level"), condition, desc) + # Pattern: check_require_if(col_expr, "field_name", ... + pattern = re.compile(r'check_require_if\([^,]+,\s*"[^"]+",\s*F\.') + assert not pattern.search(source), ( + "check_require_if still passes field name as string arg" + ) + + def test_forbid_if_no_target_name_arg(self) -> None: + """check_forbid_if must not pass the field name as a string arg.""" + source = _render(RequireForbidModel, "rf") + pattern = re.compile(r'check_forbid_if\([^,]+,\s*"[^"]+",\s*F\.') + assert not pattern.search(source), ( + "check_forbid_if still passes field name as string arg" + ) + + +class TestEnumValueInCondition: + def test_renders_valid_python(self) -> None: + source = _render(RequireIfEnumModel, "require_if_enum") + ast.parse(source) + + def test_enum_value_rendered_as_string_literal_in_column_expr(self) -> None: + source = _render(RequireIfEnumModel, "require_if_enum") + # The column expression (F.col == ...) must use the plain string value, + # not the non-parseable enum repr <_SubtypeEnum.COUNTRY: 'country'>. + # The condition description string may still contain the enum repr since + # it's only displayed in error messages (inside a quoted string literal). + assert "'country'" in source + + +class TestConditionDescriptionRendering: + """Model constraint condition descriptions are human-readable, not Python repr.""" + + def test_condition_desc_no_enum_repr(self) -> None: + source = _render(RequireIfEnumModel, "require_if_enum") + # The condition_desc string (4th arg to check_require_if) must not contain + # the non-parseable enum repr like <_SubtypeEnum.COUNTRY: 'country'> + assert "<_SubtypeEnum" not in source + + def test_condition_desc_uses_field_eq_format(self) -> None: + source = _render(RequireIfEnumModel, "require_if_enum") + # Should render as "subtype = 'country'" style (value quoted) + assert "subtype = 'country'" in source + + def test_condition_desc_with_double_quote_in_value_parseable(self) -> None: + """Condition values containing double-quotes must produce parseable output.""" + + @require_if(["admin_level"], FieldEqCondition("subtype", 'say "hi"')) + class DoubleQuoteCondModel(BaseModel): + subtype: str + admin_level: int | None = None + + source = _render(DoubleQuoteCondModel, "dq_cond") + ast.parse(source) + + +@forbid_if(["admin_level"], FieldEqCondition("subtype", "country")) +@require_if(["admin_level"], Not(FieldEqCondition("subtype", "country"))) +class RequireForbidModel(BaseModel): + subtype: str + admin_level: int | None = None + + +class TestModelConstraintFieldLabels: + """require_if/forbid_if field labels: no suffix when unique, per-field counter on collision.""" + + def test_require_if_single_constraint_no_suffix(self) -> None: + source = _render(RequireIfEnumModel, "require_if_enum") + assert "field='admin_level_required'" in source + + def test_forbid_if_single_constraint_no_suffix(self) -> None: + source = _render(RequireForbidModel, "rf") + assert "field='admin_level_forbidden'" in source + + def test_require_and_forbid_have_distinct_labels(self) -> None: + source = _render(RequireForbidModel, "rf") + assert "field='admin_level_required'" in source + assert "field='admin_level_forbidden'" in source + + def test_multiple_require_if_same_target_disambiguated(self) -> None: + """Multiple require_if on the same target get per-field numeric suffixes.""" + + @require_if(["level"], FieldEqCondition("kind", "a")) + @require_if(["level"], FieldEqCondition("kind", "b")) + class MultiRequireModel(BaseModel): + kind: str + level: int | None = None + + source = _render(MultiRequireModel, "multi_req") + labels = re.findall(r"field='(level_required[^']*)'", source) + assert len(labels) >= 2, f"Expected >=2 unique labels, got {labels}" + assert len(labels) == len(set(labels)), f"Duplicate labels: {labels}" + assert all(re.search(r"_\d+$", lbl) for lbl in labels), ( + f"Expected numeric suffixes on collision labels: {labels}" + ) + + +class TestDuplicateFunctionNames: + def test_column_and_element_level_get_unique_names(self) -> None: + """division_ids and division_ids[] should produce distinct function names.""" + col_check = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("items"), + ) + elem_check = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("items[]"), + ) + source = render_model_module("dup", [col_check, elem_check], [], []) + ast.parse(source) + func_defs = re.findall(r"^def (_\w+_check\w*)\(", source, re.MULTILINE) + assert len(func_defs) == len(set(func_defs)), ( + f"Duplicate function names: {func_defs}" + ) + + def test_same_field_different_variants_get_unique_names(self) -> None: + """class for road and class for rail should produce distinct function names.""" + road_check = Check( + descriptors=( + ExpressionDescriptor(function="check_enum", args=(["a", "b"],)), + ), + target=_path("class"), + guards=(ColumnGuard(discriminator="subtype", values=("road",)),), + ) + rail_check = Check( + descriptors=( + ExpressionDescriptor(function="check_enum", args=(["x", "y"],)), + ), + target=_path("class"), + guards=(ColumnGuard(discriminator="subtype", values=("rail",)),), + ) + source = render_model_module("dup", [road_check, rail_check], [], []) + ast.parse(source) + func_defs = re.findall(r"^def (_\w+_check\w*)\(", source, re.MULTILINE) + assert len(func_defs) == len(set(func_defs)), ( + f"Duplicate function names: {func_defs}" + ) + + +class TestFieldCheckLabelCollision: + """Field checks sharing a `(field, name)` identity get distinct labels. + + The discriminated vehicle-dimension union in `segment` emits two + field checks with the identical identity + `("...vehicle[].value", "required")` -- one per arm of the inner + union. Without a collision suffix the emitted `Check.field` is + ambiguous (it keys `suppress` matching, `explain_errors` metadata, + and the conformance test's `expected_field`). Mirror the model-check + `_N` convention: every member of a colliding group gets a suffix. + """ + + def test_colliding_required_checks_get_distinct_labels(self) -> None: + first = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("value"), + guards=(ElementGuard(discriminator="dimension", values=("axle_count",)),), + ) + second = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("value"), + guards=( + ElementGuard(discriminator="dimension", values=("height", "width")), + ), + ) + source = render_model_module("collide", [first, second], [], []) + ast.parse(source) + labels = re.findall(r"field='(value[^']*)'", source) + assert labels == ["value_0", "value_1"], labels + + def test_noncolliding_field_check_stays_bare(self) -> None: + required = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("value"), + ) + bounds = Check( + descriptors=( + ExpressionDescriptor(function="check_bounds", kwargs=(("ge", 0),)), + ), + target=_path("value"), + ) + source = render_model_module("solo", [required, bounds], [], []) + ast.parse(source) + labels = re.findall(r"field='(value[^']*)'", source) + assert labels == ["value", "value"], labels + + def test_multi_descriptor_collision_only_on_shared_name(self) -> None: + """A multi-descriptor check collides per emitted `(field, name)` row.""" + single = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("value"), + ) + multi = Check( + descriptors=( + ExpressionDescriptor(function="check_required"), + ExpressionDescriptor(function="check_bounds", kwargs=(("ge", 0),)), + ), + target=_path("value"), + ) + source = render_model_module("multi", [single, multi], [], []) + ast.parse(source) + # The two `required` rows collide (-> value_0/value_1); the lone + # `bounds` row stays bare. + required_fields = re.findall( + r"field='(value[^']*)',\n\s+name='required'", source + ) + bounds_fields = re.findall(r"field='(value[^']*)',\n\s+name='bounds'", source) + assert required_fields == ["value_0", "value_1"], required_fields + assert bounds_fields == ["value"], bounds_fields + + def test_labels_are_positional_not_identity_keyed(self) -> None: + """Row labels align to flattened `(check, desc_idx)` order. + + Collision suffixes depend only on the iteration order both + renderers share -- never on the identity of the `Check` objects. + Two value-equal but distinct checks (the cross-arm collision case) + must still each receive their own collision index. + """ + first = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("value"), + ) + second = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("value"), + ) + # A distinct copy of `first`, equal by value -- under identity + # keying this would alias `first`; positional keying keeps it + # separate. + first_copy = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("value"), + ) + labels = [row.label for row in field_check_rows([first, second, first_copy])] + assert labels == ["value_0", "value_1", "value_2"], labels + + +class TestMapPathRendering: + """MapPath targets render to map_keys_check / map_values_check.""" + + def test_map_key_renders_map_keys_check(self) -> None: + check = Check( + descriptors=( + ExpressionDescriptor( + function="check_pattern", + args=(r"^[a-z]+$",), + label="language tag", + ), + ), + target=_path("names{key}"), + ) + source = render_model_module("dictfeat", [check], [], []) + ast.parse(source) + assert 'map_keys_check("names", lambda k: check_pattern(k,' in source + + def test_map_value_renders_map_values_check(self) -> None: + check = Check( + descriptors=(ExpressionDescriptor(function="check_stripped"),), + target=_path("names.common{value}"), + ) + source = render_model_module("dictfeat", [check], [], []) + ast.parse(source) + assert 'map_values_check("names.common", lambda v: check_stripped(v))' in source + + def test_map_check_imports_helper_from_column_patterns(self) -> None: + check = Check( + descriptors=(ExpressionDescriptor(function="check_stripped"),), + target=_path("names{value}"), + ) + source = render_model_module("dictfeat", [check], [], []) + assert re.search( + r"from [.\w]*column_patterns import[\s\S]*?map_values_check", source + ) + + def test_map_check_read_columns_is_top_level_column(self) -> None: + # The map check dereferences its top-level map column (`names`), not the + # dotted struct path or the `{value}` step marker; `read_columns` is the + # granularity at which validate drops the check when the column is absent. + check = Check( + descriptors=(ExpressionDescriptor(function="check_stripped"),), + target=_path("names.common{value}"), + ) + source = render_model_module("dictfeat", [check], [], []) + # Renderer emits repr() (single quotes); ruff later normalizes. + assert "read_columns=frozenset({'names'})" in source + + +@require_any_of("x", "y") +class _ArrayElementConstrained(BaseModel): + x: str | None = None + y: str | None = None + + +class ArrayOfConstrained(BaseModel): + items: list[_ArrayElementConstrained] + + +class TestArrayModelConstraintRendering: + """Model constraints on array elements render inside array_check.""" + + def test_renders_parseable_python(self) -> None: + source = _render(ArrayOfConstrained, "arr_constrained") + ast.parse(source) + + def test_renders_array_check(self) -> None: + source = _render(ArrayOfConstrained, "arr_constrained") + assert "array_check(" in source + + def test_renders_el_field_refs(self) -> None: + source = _render(ArrayOfConstrained, "arr_constrained") + assert 'el["x"]' in source + assert 'el["y"]' in source + + def test_no_f_col_for_array_element_constraint(self) -> None: + source = _render(ArrayOfConstrained, "arr_constrained") + # The array-element model constraint should not use F.col for its field refs + assert 'F.col("x")' not in source + assert 'F.col("y")' not in source + + def test_shape_is_array(self) -> None: + source = _render(ArrayOfConstrained, "arr_constrained") + assert "CheckShape.ARRAY" in source + + def test_field_label_uses_prefix(self) -> None: + source = _render(ArrayOfConstrained, "arr_constrained") + assert "field='items[]" in source + + def test_imports_array_check(self) -> None: + source = _render(ArrayOfConstrained, "arr_constrained") + assert "array_check" in source + + +class TestVariantDiscriminatorField: + def test_variant_uses_check_discriminator_field(self) -> None: + """Variant gating should use the Guard's discriminator field, not hardcoded 'subtype'.""" + check = Check( + descriptors=( + ExpressionDescriptor(function="check_enum", args=(["x", "y"],)), + ), + target=_path("a_field"), + guards=(ColumnGuard(discriminator="kind", values=("a",)),), + ) + source = render_model_module("test_variant", [check], [], []) + ast.parse(source) + assert 'F.col("kind")' in source + assert 'F.col("subtype")' not in source + + +@require_any_of("a", "b") +class _NestedConstrainedStruct(BaseModel): + a: str | None = None + b: str | None = None + + +class _ArrayElementWithNestedConstraint(BaseModel): + nested: _NestedConstrainedStruct + + +class ArrayOfNestedConstrained(BaseModel): + items: list[_ArrayElementWithNestedConstraint] + + +class TestVariantGatedArrayLambdaScope: + """Variant gating for ARRAY-shaped nodes must be inside the lambda, not wrapping it.""" + + @pytest.fixture(scope="class") + def rendered_source(self) -> str: + class _Base(BaseModel): + kind: str + + class _TypeA(_Base): + kind: Literal["a"] = "a" + a_field: str + + class _TypeB(_Base): + kind: Literal["b"] = "b" + + _Union = Annotated[ + Union[_TypeA, _TypeB], # noqa: UP007 + FieldInfo(discriminator="kind"), + ] + + class _Wrapper(BaseModel): + items: list[_Union] + + return _render(_Wrapper, "wrapper") + + def test_parseable(self, rendered_source: str) -> None: + ast.parse(rendered_source) + + def test_variant_gating_inside_lambda(self, rendered_source: str) -> None: + """el['kind'] must appear inside the lambda body, not outside array_check.""" + lines = rendered_source.splitlines() + for i, line in enumerate(lines): + if "array_check(" in line and i > 0: + preceding = lines[i - 1].strip() + assert not preceding.startswith("F.when("), ( + f"array_check wrapped by F.when at line {i}: {lines[i - 1]!r}" + ) + + lambda_found = False + el_kind_inside_lambda = False + for line in lines: + if "lambda el:" in line: + lambda_found = True + if lambda_found and 'el["kind"]' in line: + el_kind_inside_lambda = True + break + + assert lambda_found, "No lambda el: found in generated source" + assert el_kind_inside_lambda, ( + 'el["kind"] never appears after lambda el: — variant gating is outside lambda scope' + ) + + +class TestTopLevelVariantGatedArray: + """When the array column itself is variant-conditional, discriminator wraps array_check.""" + + @pytest.fixture(scope="class") + def surface_check(self) -> Check: + """ARRAY check with top-level discriminator -- surface only exists for subtype='a'.""" + return Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("surface[]"), + guards=(ColumnGuard(discriminator="subtype", values=("a",)),), + ) + + @pytest.fixture(scope="class") + def surface_value_check(self) -> Check: + """ARRAY check with leaf path and top-level discriminator.""" + return Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("surface[].value"), + guards=(ColumnGuard(discriminator="subtype", values=("a",)),), + ) + + def test_parseable(self, surface_check: Check) -> None: + source = render_model_module("test", [surface_check], [], []) + ast.parse(source) + + def test_discriminator_uses_f_col(self, surface_check: Check) -> None: + """Top-level discriminator must reference F.col, not el[...].""" + source = render_model_module("test", [surface_check], [], []) + assert 'F.col("subtype")' in source, ( + "Top-level discriminator must use F.col, not el[...]" + ) + assert 'el["subtype"]' not in source, ( + 'el["subtype"] found -- discriminator placed inside lambda' + ) + + def test_f_when_wraps_array_check(self, surface_check: Check) -> None: + """F.when must wrap the array_check call, not the lambda body.""" + source = _render_check_function(surface_check) + # F.when must appear before array_check in the expression. + f_when_pos = source.find("F.when(") + array_check_pos = source.find("array_check(") + assert f_when_pos != -1, "F.when not found in output" + assert array_check_pos != -1, "array_check not found in output" + assert f_when_pos < array_check_pos, ( + f"F.when (pos {f_when_pos}) must appear before array_check (pos {array_check_pos})" + ) + + def test_no_el_discriminator_in_lambda(self, surface_value_check: Check) -> None: + """el['subtype'] must not appear even with leaf path -- subtype is top-level.""" + source = render_model_module("test", [surface_value_check], [], []) + assert 'el["subtype"]' not in source, ( + 'el["subtype"] found -- top-level discriminator must not appear inside lambda' + ) + + def test_leaf_path_check_parseable(self, surface_value_check: Check) -> None: + source = render_model_module("test", [surface_value_check], [], []) + ast.parse(source) + + +class TestNestedStructModelConstraintRendering: + """Nested struct model constraints inside array elements use chained el accessors.""" + + def test_renders_parseable_python(self) -> None: + source = _render(ArrayOfNestedConstrained, "nested_constrained") + ast.parse(source) + + def test_chained_struct_accessor(self) -> None: + source = _render(ArrayOfNestedConstrained, "nested_constrained") + assert 'el["nested"]["a"]' in source + assert 'el["nested"]["b"]' in source + + def test_no_direct_el_access(self) -> None: + """Should NOT produce el["a"] — must go through nested struct.""" + source = _render(ArrayOfNestedConstrained, "nested_constrained") + # el["a"] without ["nested"] prefix should not appear + lines = source.split("\n") + for line in lines: + if 'el["a"]' in line and '["nested"]' not in line: + pytest.fail(f'Found bare el["a"] without struct prefix: {line}') + + +class TestRenderNestedArrayCheckStructure: + """_render_check_function emits correct nested_array_check / lambda structure.""" + + def test_render_nested_array_check(self) -> None: + check = Check( + descriptors=( + ExpressionDescriptor(function="check_bounds", kwargs=(("ge", 0),)), + ), + target=_path("items[].things[].value"), + ) + source = _render_check_function(check) + assert "nested_array_check" in source + assert "lambda el" in source + assert "lambda inner" in source + assert 'el["things"]' in source + assert 'check_bounds(inner["value"],' in source + + def test_render_variant_expr_in_nested_array_top_level_disc(self) -> None: + """Top-level discriminator wraps nested_array_check in F.when(F.col(...)).""" + check = Check( + descriptors=( + ExpressionDescriptor(function="check_enum", args=(["m", "km"],)), + ), + target=_path("items[].things[].unit"), + guards=(ColumnGuard(discriminator="kind", values=("a", "b")),), + ) + source = _render_check_function(check) + assert "nested_array_check" in source + assert 'F.col("kind").isin(' in source + + def test_render_variant_expr_in_nested_array_element_disc(self) -> None: + """Element-level discriminator gates inside the inner lambda.""" + check = Check( + descriptors=( + ExpressionDescriptor(function="check_enum", args=(["m", "km"],)), + ), + target=_path("items[].things[].unit"), + guards=(ElementGuard(discriminator="kind", values=("a", "b")),), + ) + source = _render_check_function(check) + assert "nested_array_check" in source + assert 'F.col("kind")' not in source + assert 'inner["kind"]' in source + + +@require_any_of("a", "b") +class _DoubleNestedConstrainedElement(BaseModel): + a: str | None = None + b: str | None = None + + +class _OuterArrayElement(BaseModel): + things: list[_DoubleNestedConstrainedElement] + + +class _DoubleNestedModel(BaseModel): + items: list[_OuterArrayElement] + + +class TestDoubleNestedArrayModelConstraintRendering: + """Model constraints on list[] inside another array render nested_array_check.""" + + def test_renders_parseable_python(self) -> None: + source = _render(_DoubleNestedModel, "double_nested") + ast.parse(source) + + def test_uses_nested_array_check(self) -> None: + source = _render(_DoubleNestedModel, "double_nested") + assert "nested_array_check" in source + + def test_inner_lambda_uses_inner_variable(self) -> None: + source = _render(_DoubleNestedModel, "double_nested") + assert 'inner["a"]' in source + assert 'inner["b"]' in source + + def test_outer_lambda_navigates_to_inner_array(self) -> None: + source = _render(_DoubleNestedModel, "double_nested") + assert 'el["things"]' in source + + +class TestMultiLevelNestedArrayRendering: + """Rendering of deeply nested array checks (2+ inner levels).""" + + def test_two_inner_levels_produces_double_nesting(self) -> None: + """list[list[list[Struct]]].field -> nested(nested(array_check)).""" + check = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("items[][][].value"), + ) + source = _render_node(check) + # Three IterateArrays -> 1 outer nested_array_check + 1 intermediate + # nested_array_check + 1 innermost array_check = 2 nested_array_check calls. + assert source.count("nested_array_check(") == 2 + assert "lambda el:" in source + assert "lambda el2:" in source # intermediate level + assert "lambda inner:" in source # innermost + assert 'check_required(inner["value"])' in source + + def test_two_inner_levels_with_struct_path(self) -> None: + """Intermediate level with struct navigation.""" + check = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("outer[].mid[][].leaf"), + ) + source = _render_node(check) + assert 'el["mid"]' in source + assert source.count("nested_array_check(") == 2 + + def test_model_constraint_with_two_inner_levels(self) -> None: + """Model constraint at depth 3 uses double-nested wrapping.""" + check = ModelCheck( + descriptor=RequireAnyOf(field_names=("a", "b")), + target=_path("items[][][]"), + ) + source = _render_model_node(check) + assert source.count("nested_array_check(") == 2 + assert "lambda el:" in source + assert "lambda el2:" in source + assert "array_check(" in source + + def test_variant_gating_only_at_innermost_level(self) -> None: + """Variant values on a multi-level check with element guard apply at innermost.""" + check = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("items[][][].value"), + guards=(ElementGuard(discriminator="kind", values=("type_a",)),), + ) + source = _render_node(check) + # Variant gating appears at the innermost level. + assert 'inner["kind"]' in source + + +class TestGatedScalarRendering: + """Gated check_required wraps expression in F.when(gate.isNotNull(), ...).""" + + @pytest.fixture + def gated_check(self) -> Check: + return Check( + descriptors=( + ExpressionDescriptor(function="check_required", gate=_path("inner")), + ), + target=_path("inner.value"), + ) + + def test_gated_scalar_has_when_wrapping(self, gated_check: Check) -> None: + source = _render_node(gated_check) + assert 'F.col("inner").isNotNull()' in source + assert "check_required" in source + assert "F.when(" in source + + def test_gated_scalar_is_parseable(self, gated_check: Check) -> None: + source = _render_node(gated_check) + ast.parse(source) + + def test_ungated_scalar_unchanged(self) -> None: + check = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("value"), + ) + source = _render_node(check) + assert "isNotNull" not in source + assert "check_required" in source + + +class _NullableNestedElement(BaseModel): + value: str + + +class _ElementWithNullableStruct(BaseModel): + nested: _NullableNestedElement | None = None + + +class _ArrayWithNullableStruct(BaseModel): + items: list[_ElementWithNullableStruct] + + +class TestGatedFullModelRendering: + def test_gated_array_descriptor_is_parseable(self) -> None: + source = _render(_ArrayWithNullableStruct, "arr") + ast.parse(source) + + def test_gated_array_descriptor_has_element_gate(self) -> None: + source = _render(_ArrayWithNullableStruct, "arr") + assert 'el["nested"].isNotNull()' in source + assert "check_required" in source + + def test_model_with_nullable_parent_is_parseable(self) -> None: + class Inner(BaseModel): + value: str + + class Outer(BaseModel): + inner: Inner | None = None + + source = _render(Outer, "outer") + ast.parse(source) + assert "isNotNull" in source + assert "check_required" in source + + +class TestGatedArrayRendering: + """Gated check_required in array context uses element accessor for gate.""" + + @pytest.fixture + def element_gated_check(self) -> Check: + return Check( + descriptors=( + ExpressionDescriptor( + function="check_required", gate=_path("items[].nested") + ), + ), + target=_path("items[].nested.mode"), + ) + + def test_gated_array_has_element_gate(self, element_gated_check: Check) -> None: + source = _render_node(element_gated_check) + assert 'el["nested"].isNotNull()' in source + assert "check_required" in source + assert "F.when(" in source + + def test_gated_array_is_parseable(self, element_gated_check: Check) -> None: + source = _render_node(element_gated_check) + ast.parse(source) + + def test_column_level_gate_on_array_target_raises(self) -> None: + """A column-level gate on an ArrayPath target is not produced by check_builder.""" + check = Check( + descriptors=( + ExpressionDescriptor( + function="check_required", gate=_path("perspectives") + ), + ), + target=_path("perspectives.countries[]"), + ) + with pytest.raises(AssertionError, match="column-level gate"): + _render_node(check) + + def test_nested_array_gate_applied_at_outermost_lambda(self) -> None: + """Gate on a nested_array_check wraps the el lambda body, not inner.""" + check = Check( + descriptors=( + ExpressionDescriptor( + function="check_required", gate=_path("rules[].perspectives") + ), + ), + target=_path("rules[].perspectives.countries[]"), + ) + source = _render_node(check) + ast.parse(source) + assert "nested_array_check(" in source + # Gate must be on el (the rule struct), not inner (the country string). + assert 'el["perspectives"].isNotNull()' in source + assert "inner[" not in source + + +@require_any_of("a", "b") +class _OptionalSubModel(BaseModel): + a: str | None = None + b: str | None = None + + +class _ElementWithOptional(BaseModel): + nested: _OptionalSubModel | None = None + + +class _ArrayWithOptionalSubModel(BaseModel): + items: list[_ElementWithOptional] + + +class TestGatedModelConstraintRendering: + """ModelCheck with gate wraps the constraint in F.when(.isNotNull(), ...).""" + + def test_gated_model_check_wraps_in_f_when(self) -> None: + """A gated ModelCheck on items[].nested emits F.when(el['nested'].isNotNull(), ...).""" + check = ModelCheck( + descriptor=RequireAnyOf(field_names=("a", "b")), + target=_path("items[].nested"), + gate=_path("items[].nested"), + ) + source = _render_model_node(check) + assert 'el["nested"].isNotNull()' in source + assert "check_require_any_of" in source + assert "F.when(" in source + + def test_gated_model_check_is_parseable(self) -> None: + check = ModelCheck( + descriptor=RequireAnyOf(field_names=("a", "b")), + target=_path("items[].nested"), + gate=_path("items[].nested"), + ) + source = _render_model_node(check) + ast.parse(source) + + def test_ungated_model_check_no_f_when(self) -> None: + """A ModelCheck without gate does NOT emit isNotNull wrapping.""" + check = ModelCheck( + descriptor=RequireAnyOf(field_names=("x", "y")), + target=_path("items[]"), + gate=None, + ) + source = _render_model_node(check) + assert "isNotNull" not in source + assert "check_require_any_of" in source + + def test_full_render_optional_sub_model_has_when_guard(self) -> None: + """End-to-end: rendering _ArrayWithOptionalSubModel emits the isNotNull guard.""" + source = _render(_ArrayWithOptionalSubModel, "arr_optional_sub") + assert 'el["nested"].isNotNull()' in source + + def test_full_render_optional_sub_model_parseable(self) -> None: + source = _render(_ArrayWithOptionalSubModel, "arr_optional_sub") + ast.parse(source) + + def test_gated_model_check_assertion_on_non_array_target(self) -> None: + """A gate paired with a non-ArrayPath target raises AssertionError.""" + check = ModelCheck( + descriptor=RequireAnyOf(field_names=("a", "b")), + target=ScalarPath(), + gate=_path("items[].nested"), + ) + with pytest.raises(AssertionError, match="gate.*non-ArrayPath"): + _render_model_node(check) + + +class TestMapValueModelRendering: + """Render `dict[K, Model]` value-model checks inside a map lambda. + + The map's values are iterated like an array: a value-model field check + renders `map_values_check("col", lambda v: check(v["field"]))`, and a + value-model constraint renders `map_values_check("col", lambda v: + check_require_any_of([v["a"], v["b"]], ...))`. Both mirror the + `array_check` rendering of a `list[Model]` element. + """ + + def _field_check(self, model_cls: type[BaseModel], function: str) -> Check: + field_checks, _ = build_checks(spec_for_model(model_cls)) + for check in field_checks: + if any(d.function == function for d in check.descriptors): + return check + raise AssertionError(f"no field check with {function}") + + def _model_check(self, model_cls: type[BaseModel]) -> ModelCheck: + _, model_checks = build_checks(spec_for_model(model_cls)) + assert len(model_checks) == 1, model_checks + return model_checks[0] + + def test_value_field_check_renders_map_values_lambda(self) -> None: + check = self._field_check(MapValueFieldModel, "check_string_min_length") + rows = field_check_rows([check]) + sources = [ + _render_check_function_string(_render_check_function_context(row)) + for row in rows + ] + assert any( + 'map_values_check("items", lambda v: check_string_min_length(v["label"], 1))' + in s + for s in sources + ), sources + + def test_value_model_constraint_renders_map_values_lambda(self) -> None: + # Asserts the raw renderer form: field names render via repr (single + # quotes); ruff normalizes to double quotes downstream. + check = self._model_check(MapValueConstraintModel) + source = _render_model_node(check) + assert ( + 'map_values_check("subs", lambda v: ' + "check_require_any_of([v[\"foo\"], v[\"bar\"]], ['foo', 'bar']))" + ) in source, source + + def test_full_module_parseable_with_map_value_field(self) -> None: + source = _render(MapValueFieldModel, "map_field") + ast.parse(source) + assert "map_values_check(" in source + + def test_full_module_parseable_with_map_value_constraint(self) -> None: + source = _render(MapValueConstraintModel, "map_con") + ast.parse(source) + assert "map_values_check(" in source + + def test_model_constraint_func_name_prefixes_map_column(self) -> None: + # Mirrors the ArrayPath naming so distinct map columns yield distinct + # generated function names rather than colliding on `__`. + check = self._model_check(MapValueConstraintModel) + source = _render_model_node(check) + assert "def _subs_check_require_any_of_0_check()" in source, source + + def test_forbid_if_value_constraint_renders_map_values_lambda(self) -> None: + check = self._model_check(MapValueForbidIfModel) + source = _render_model_node(check) + assert ( + 'map_values_check("subs", lambda v: ' + 'check_forbid_if(v["extra"], v["kind"] == \'basic\', "kind = \'basic\'"))' + ) in source, source + + +class TestLeafQualifiedConditionRef: + """A require_if/forbid_if condition reached through a non-empty leaf + keeps the leaf. + + The target field ref and the condition field ref navigate the same + struct leaf; rendering the condition without the leaf references a + wrong column (a top-level field of the iterated element instead of the + nested struct's field). The leaf is non-empty for a constrained model + reached below an iterated container -- both `list[Model]` and + `dict[K, Model]`. + """ + + def _require_if_check(self, model_cls: type[BaseModel]) -> ModelCheck: + _, model_checks = build_checks(spec_for_model(model_cls)) + matches = [c for c in model_checks if isinstance(c.descriptor, RequireIf)] + assert len(matches) == 1, model_checks + return matches[0] + + def test_map_value_require_if_condition_keeps_leaf(self) -> None: + source = _render_model_node(self._require_if_check(MapValueRequireIfModel)) + assert 'v["inner"]["admin_level"]' in source, source + assert 'v["inner"]["subtype"] ==' in source, source + + def test_array_require_if_condition_keeps_leaf(self) -> None: + source = _render_model_node(self._require_if_check(ArrayValueRequireIfModel)) + assert 'el["inner"]["admin_level"]' in source, source + assert 'el["inner"]["subtype"] ==' in source, source + + +class _UrlOrEmptyRender(BaseModel): + """Required `HttpUrl | Literal[""]` -- the literal bypasses the URL checks.""" + + data_url: HttpUrl | Literal[""] + + +class TestLiteralAlternativesRendering: + """A descriptor carrying allow_literals renders an except_literals wrapper.""" + + def test_url_checks_wrapped_in_except_literals(self) -> None: + source = _render(_UrlOrEmptyRender) + # Both content checks are wrapped; the literal value is threaded in. + assert 'except_literals(F.col("data_url"), check_url_format(' in source, source + assert 'except_literals(F.col("data_url"), check_url_length(' in source, source + # py_literal emits the pre-ruff form (single quotes); ruff normalizes later. + assert ", ['']" in source, source + + def test_except_literals_imported(self) -> None: + source = _render(_UrlOrEmptyRender) + assert "except_literals" in source + # check_required is not wrapped: it is not threaded with allow_literals. + assert 'except_literals(F.col("data_url"), check_required(' not in source diff --git a/packages/overture-schema-codegen/tests/test_pyspark_scaffold.py b/packages/overture-schema-codegen/tests/test_pyspark_scaffold.py new file mode 100644 index 000000000..dbff1f009 --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_pyspark_scaffold.py @@ -0,0 +1,383 @@ +"""Tests for sparse path scaffold generation.""" + +import copy +from dataclasses import replace +from typing import Any + +import pytest +from codegen_test_support import ( + FeatureWithRequiredUrl, + discover_feature, + spec_for_model, +) +from overture.schema.codegen.extraction.specs import ModelSpec, UnionSpec +from overture.schema.codegen.pyspark.check_builder import build_checks +from overture.schema.codegen.pyspark.check_ir import ( + Check, + ColumnGuard, + ElementGuard, + ModelCheck, +) +from overture.schema.codegen.pyspark.constraint_dispatch import RequireAnyOf +from overture.schema.codegen.pyspark.test_data.base_row import ( + generate_arm_rows, + generate_base_row, +) +from overture.schema.codegen.pyspark.test_data.scaffold import ( + generate_model_scaffold, + generate_scaffold, + leaf_list_depth, +) +from overture.schema.system.field_path import ArrayPath, parse +from pydantic import TypeAdapter + +_path = parse + + +def _deep_merge(base: dict, scaffold: dict) -> dict: + """Merge `scaffold` onto a deep copy of `base` (harness `deep_merge` semantics). + + Dicts merge recursively; every other value (including lists) replaces the + base value. Mirrors `overture.schema.pyspark`'s conformance harness so the + validated row matches what the generated suite builds. + """ + result = copy.deepcopy(base) + for key, value in scaffold.items(): + if key in result and isinstance(result[key], dict) and isinstance(value, dict): + result[key] = _deep_merge(result[key], value) + else: + result[key] = copy.deepcopy(value) + return result + + +def _check_belongs_to_arm(check: Check, arm: str) -> bool: + """Whether a field check applies to a union arm (every `ColumnGuard` admits it).""" + return all(arm in g.values for g in check.guards if isinstance(g, ColumnGuard)) + + +@pytest.fixture(scope="module") +def connector_spec() -> ModelSpec: + return discover_feature("Connector") + + +@pytest.fixture(scope="module") +def division_area_spec() -> ModelSpec: + return discover_feature("DivisionArea") + + +@pytest.fixture(scope="module") +def segment_spec() -> ModelSpec: + return discover_feature("Segment") + + +class TestLeafListDepth: + def test_leaf_list_depth(self) -> None: + """leaf_list_depth returns unaccounted-for list depth.""" + spec = spec_for_model(FeatureWithRequiredUrl) + # Scalar field inside array struct — no extra wrapping + assert leaf_list_depth(_path("datasets[].url"), spec) == 0 + # List field without trailing array marker — needs wrapping + assert leaf_list_depth(_path("datasets[].download_urls"), spec) == 1 + # List field with array marker means element-level access — no wrapping + assert leaf_list_depth(_path("datasets[].download_urls[]"), spec) == 0 + + +class TestNestedListUrlField: + """Scaffold for FeatureWithRequiredUrl handles nested list[HttpUrl] fields.""" + + def test_nested_list_url_field_single_depth(self) -> None: + """list[HttpUrl] scaffold should be single-depth, not double-wrapped.""" + spec = spec_for_model(FeatureWithRequiredUrl) + field_nodes, _ = build_checks(spec) + url_nodes = [n for n in field_nodes if "download_urls" in str(n.target)] + assert url_nodes, "Expected check nodes for download_urls" + for node in url_nodes: + scaffold = generate_scaffold(node, spec) + if "datasets" in scaffold: + entry = scaffold["datasets"][0] + if "download_urls" in entry: + val = entry["download_urls"] + assert isinstance(val, list) + assert all(isinstance(v, str) for v in val), ( + f"Expected list[str], got nested structure: {val!r}" + ) + + +class TestGenerateScaffoldConnector: + """Scaffold for Connector — simple top-level and one-level-nested fields.""" + + def test_required_top_level_field_produces_empty_scaffold( + self, connector_spec: ModelSpec + ) -> None: + """Required top-level fields exist in base row; scaffold adds nothing.""" + field_nodes, _ = build_checks(connector_spec) + id_node = next(n for n in field_nodes if n.target == _path("id")) + scaffold = generate_scaffold(id_node, connector_spec) + assert scaffold == {} + + def test_optional_top_level_field_produces_scaffold( + self, connector_spec: ModelSpec + ) -> None: + """Optional fields absent from base row get a valid scaffold value.""" + field_nodes, _ = build_checks(connector_spec) + node = next( + n + for n in field_nodes + if n.target == _path("sources") + and any(d.function == "check_array_min_length" for d in n.descriptors) + ) + scaffold = generate_scaffold(node, connector_spec) + assert "sources" in scaffold + assert isinstance(scaffold["sources"], list) + assert len(scaffold["sources"]) >= 1 + + def test_array_nested_field_builds_path(self, connector_spec: ModelSpec) -> None: + """sources[].property needs a sources array with one element.""" + field_nodes, _ = build_checks(connector_spec) + node = next(n for n in field_nodes if n.target == _path("sources[].property")) + scaffold = generate_scaffold(node, connector_spec) + assert "sources" in scaffold + assert isinstance(scaffold["sources"], list) + assert len(scaffold["sources"]) == 1 + elem = scaffold["sources"][0] + # Required sibling 'dataset' populated + assert "dataset" in elem + + def test_scaffold_is_dict(self, connector_spec: ModelSpec) -> None: + field_nodes, _ = build_checks(connector_spec) + for node in field_nodes: + scaffold = generate_scaffold(node, connector_spec) + assert isinstance(scaffold, dict) + + +class TestGenerateScaffoldSegment: + """Scaffold for Segment — deeply nested arrays and discriminators.""" + + def test_suffixed_nested_leaf_uses_actual_field_name( + self, segment_spec: ModelSpec + ) -> None: + """Column-level checks share the structural path with the real field.""" + field_nodes, _ = build_checks(segment_spec) + node = next( + n + for n in field_nodes + if n.target == _path("access_restrictions[].when.mode") + and any(d.function == "check_array_min_length" for d in n.descriptors) + ) + scaffold = generate_scaffold(node, segment_spec) + assert "access_restrictions" in scaffold + when = scaffold["access_restrictions"][0]["when"] + assert "mode" in when, f"Expected 'mode', got keys: {list(when.keys())}" + assert "mode_min_length" not in when + + def test_deeply_nested_array_path(self, segment_spec: ModelSpec) -> None: + """speed_limits[].when.vehicle[].dimension builds full nesting.""" + field_nodes, _ = build_checks(segment_spec) + node = next( + n + for n in field_nodes + if n.target == _path("speed_limits[].when.vehicle[].dimension") + ) + scaffold = generate_scaffold(node, segment_spec) + assert "speed_limits" in scaffold + sl_elem = scaffold["speed_limits"][0] + assert "when" in sl_elem + when = sl_elem["when"] + assert "vehicle" in when + assert isinstance(when["vehicle"], list) + assert len(when["vehicle"]) == 1 + + def test_element_guard_discriminator_set(self, segment_spec: ModelSpec) -> None: + """Checks with an `ElementGuard` set the discriminator value in the scaffold.""" + field_checks, _ = build_checks(segment_spec) + # Find a speed_limits check with an ElementGuard. + check = next( + c + for c in field_checks + if any(isinstance(g, ElementGuard) for g in c.guards) + and "speed_limits" in str(c.target) + ) + scaffold = generate_scaffold(check, segment_spec) + # Walk to the innermost array element where the discriminator lives. + assert "speed_limits" in scaffold + sl_elem = scaffold["speed_limits"][0] + when = sl_elem["when"] + vehicle_elem = when["vehicle"][0] + element_guard = next(g for g in check.guards if isinstance(g, ElementGuard)) + assert element_guard.discriminator in vehicle_elem + assert vehicle_elem[element_guard.discriminator] == element_guard.values[0] + + def test_column_variant_does_not_appear_inside_scaffold( + self, segment_spec: ModelSpec + ) -> None: + """`ColumnGuard`s don't set discriminator inside the scaffold dict.""" + field_checks, _ = build_checks(segment_spec) + # Find a check whose only guard is a ColumnGuard (no ElementGuard). + check = next( + c + for c in field_checks + if c.guards + and not any(isinstance(g, ElementGuard) for g in c.guards) + and "speed_limits[]." in str(c.target) + ) + scaffold = generate_scaffold(check, segment_spec) + # The column-level discriminator is NOT set in the scaffold -- + # it belongs at the row level, which the base row handles. + assert isinstance(scaffold, dict) + + def test_multiple_element_guards_raises(self, segment_spec: ModelSpec) -> None: + """The check_ir invariant allows at most one `ElementGuard` per Check. + + Multiple guards would indicate the gate composition rule changed + without updating the scaffold, so the scaffold raises rather than + silently dropping all but the first. + """ + field_checks, _ = build_checks(segment_spec) + check = next( + c + for c in field_checks + if any(isinstance(g, ElementGuard) for g in c.guards) + ) + bogus = replace( + check, + guards=( + *check.guards, + ElementGuard(discriminator="other_field", values=("other_value",)), + ), + ) + with pytest.raises(NotImplementedError, match="ElementGuards"): + generate_scaffold(bogus, segment_spec) + + +# Models whose scaffolds must merge onto a base row to form a valid instance. +# Spans a union with a union-in-array (`Segment`'s `when.vehicle[]`), record +# specs with `require_any_of` and optional nested-model arrays, a map field +# (`Infrastructure.source_tags`), and `list[list[...]]` arrays +# (`Division.hierarchies[][]`, so iter_count>1 wrapping is covered). The +# conformance suite only asserts each scenario's own expected violation is +# absent from its valid row -- whole-row validity of a scaffold is checked here, +# so a model-specific scaffold defect can't hide behind it. +_VALID_ROW_MODELS = [ + "Segment", + "Connector", + "Division", + "DivisionArea", + "DivisionBoundary", + "Place", + "Building", + "BuildingPart", + "Address", + "Infrastructure", + "Land", + "LandCover", + "LandUse", + "Water", + "Bathymetry", +] + + +def _base_rows_and_adapter(spec: ModelSpec) -> tuple[dict[str, dict[str, Any]], Any]: + """Return per-arm base rows and a Pydantic adapter for a spec. + + A `UnionSpec` yields one base row per discriminator arm, validated against + the union annotation. A record spec yields a single row keyed by `""` (a + sentinel arm carrying no `ColumnGuard`, so `_check_belongs_to_arm` admits + every record-spec check) and validates against the source class. + """ + if isinstance(spec, UnionSpec): + return generate_arm_rows(spec), TypeAdapter(spec.source_annotation) + assert spec.source_type is not None + return {"": generate_base_row(spec)}, TypeAdapter(spec.source_type) + + +class TestScaffoldsProduceValidRows: + """Ground truth for finding #1: a scaffold merged onto the base row is valid. + + The conformance harness builds the `::valid` row as + `deep_merge(base_row, scaffold)` with no mutation, then asserts the check + does not fire. That assertion is only meaningful when the merged row is a + genuinely valid instance -- otherwise unrelated `required` / + `require_any_of` violations (or a vacuous, target-absent row) let a check + that wrongly rejects a valid value ship green. These tests validate the + merged row against the Pydantic schema directly: the scaffold must reach + the target while keeping every model on the path valid. + """ + + @pytest.fixture(scope="module", params=_VALID_ROW_MODELS) + def model_case( + self, request: pytest.FixtureRequest + ) -> tuple[ModelSpec, dict[str, dict[str, Any]], Any]: + spec = discover_feature(request.param) + arm_rows, adapter = _base_rows_and_adapter(spec) + return spec, arm_rows, adapter + + def test_field_scaffolds_validate( + self, model_case: tuple[ModelSpec, dict[str, dict[str, Any]], Any] + ) -> None: + spec, arm_rows, adapter = model_case + field_checks, _ = build_checks(spec) + for check in field_checks: + scaffold = generate_scaffold(check, spec) + for arm, base in arm_rows.items(): + if not _check_belongs_to_arm(check, arm): + continue + adapter.validate_python(_deep_merge(base, scaffold)) + + def test_model_scaffolds_validate( + self, model_case: tuple[ModelSpec, dict[str, dict[str, Any]], Any] + ) -> None: + spec, arm_rows, adapter = model_case + _, model_checks = build_checks(spec) + for check in model_checks: + scaffold = generate_model_scaffold(check, spec) + for arm, base in arm_rows.items(): + if not (check.arm is None or check.arm == arm): + continue + adapter.validate_python(_deep_merge(base, scaffold)) + + +class TestGenerateModelScaffold: + def test_top_level_model_constraint_produces_empty_scaffold( + self, division_area_spec: ModelSpec + ) -> None: + """Model constraints at the top level need no nesting.""" + _, model_nodes = build_checks(division_area_spec) + assert model_nodes, "DivisionArea should have model constraints" + node = model_nodes[0] + scaffold = generate_model_scaffold(node, division_area_spec) + assert isinstance(scaffold, dict) + + def test_map_value_model_constraint_produces_empty_scaffold( + self, connector_spec: ModelSpec + ) -> None: + """A `dict[K, Model]` value-model constraint needs no scaffold. + + The mutation (`map_path=`) owns map navigation -- it corrupts the + base row's single map entry in place, or stubs one when the map is + absent. A dict scaffold can't replace a base-row map entry under + deep_merge's recursive dict merge, so {} is correct, not the + row-root-mutation bug. + """ + mc = ModelCheck( + descriptor=RequireAnyOf(field_names=("foo", "bar")), + target=_path("subs{value}"), + ) + assert generate_model_scaffold(mc, connector_spec) == {} + + def test_array_nested_model_constraint_builds_path( + self, segment_spec: ModelSpec + ) -> None: + """Model constraints inside arrays build the array path.""" + _, model_checks = build_checks(segment_spec) + if not model_checks: + pytest.skip("Segment has no model constraints") + # Find one with an array target. + nested = [c for c in model_checks if isinstance(c.target, ArrayPath)] + if not nested: + pytest.skip("No nested model constraints found") + check = nested[0] + scaffold = generate_model_scaffold(check, segment_spec) + assert isinstance(scaffold, dict) + # The scaffold should contain the column root (top-level column name). + assert isinstance(check.target, ArrayPath) + assert check.target.array_chunks[0][1] in scaffold diff --git a/packages/overture-schema-codegen/tests/test_pyspark_schema_builder.py b/packages/overture-schema-codegen/tests/test_pyspark_schema_builder.py new file mode 100644 index 000000000..c848d88b4 --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_pyspark_schema_builder.py @@ -0,0 +1,213 @@ +"""Tests for schema_builder.""" + +from enum import Enum + +import pytest +from codegen_test_support import spec_for_model +from overture.schema.codegen.extraction.field import Primitive +from overture.schema.codegen.extraction.specs import ( + AnnotatedField, + FieldSpec, + UnionSpec, +) +from overture.schema.codegen.pyspark.schema_builder import SchemaField, build_schema +from overture.schema.divisions import DivisionArea +from pydantic import BaseModel, Field + + +class SimpleModel(BaseModel): + name: str + count: int = Field(ge=0) + + +class TestPrimitiveFields: + @pytest.fixture + def fields(self) -> list[SchemaField]: + return build_schema(spec_for_model(SimpleModel)) + + def test_string_field_maps_to_string_type(self, fields: list[SchemaField]) -> None: + name_field = next(f for f in fields if f.name == "name") + assert name_field.type_expr == "StringType()" + + def test_int_field_maps_to_long_type(self, fields: list[SchemaField]) -> None: + count_field = next(f for f in fields if f.name == "count") + assert count_field.type_expr == "LongType()" + + +class NestedModel(BaseModel): + value: str + count: int + + +class ContainerModel(BaseModel): + item: NestedModel | None = None + + +class TestNestedModel: + @pytest.fixture + def fields(self) -> list[SchemaField]: + return build_schema(spec_for_model(ContainerModel)) + + def test_nested_model_emits_struct_type(self, fields: list[SchemaField]) -> None: + item_field = next(f for f in fields if f.name == "item") + assert item_field.type_expr.startswith("StructType([") + + def test_nested_struct_contains_subfields(self, fields: list[SchemaField]) -> None: + item_field = next(f for f in fields if f.name == "item") + assert 'StructField("value"' in item_field.type_expr + assert 'StructField("count"' in item_field.type_expr + + +class ListModel(BaseModel): + tags: list[str] + counts: list[int] | None = None + + +class TestListFields: + @pytest.fixture + def fields(self) -> list[SchemaField]: + return build_schema(spec_for_model(ListModel)) + + def test_list_str_maps_to_array_string(self, fields: list[SchemaField]) -> None: + tags_field = next(f for f in fields if f.name == "tags") + assert tags_field.type_expr == "ArrayType(StringType(), True)" + + def test_optional_list_int_maps_to_array_long( + self, fields: list[SchemaField] + ) -> None: + counts_field = next(f for f in fields if f.name == "counts") + assert counts_field.type_expr == "ArrayType(LongType(), True)" + + +class DictModel(BaseModel): + labels: dict[str, str] | None = None + + +class TestDictFields: + @pytest.fixture + def fields(self) -> list[SchemaField]: + return build_schema(spec_for_model(DictModel)) + + def test_dict_str_str_maps_to_map_type(self, fields: list[SchemaField]) -> None: + labels_field = next(f for f in fields if f.name == "labels") + assert labels_field.type_expr == "MapType(StringType(), StringType(), True)" + + +class TestDivisionAreaSchema: + @pytest.fixture(scope="class") + def fields(self) -> list[SchemaField]: + return build_schema(spec_for_model(DivisionArea)) + + def test_id_field_is_string_type(self, fields: list[SchemaField]) -> None: + id_field = next(f for f in fields if f.name == "id") + assert id_field.type_expr == "StringType()" + + def test_geometry_field_is_binary_type(self, fields: list[SchemaField]) -> None: + geom_field = next(f for f in fields if f.name == "geometry") + assert geom_field.type_expr == "BinaryType()" + + def test_bbox_emits_shared_struct_ref(self, fields: list[SchemaField]) -> None: + bbox_field = next(f for f in fields if f.name == "bbox") + assert bbox_field.type_expr == "BBOX_STRUCT" + + def test_version_is_integer_type(self, fields: list[SchemaField]) -> None: + ver_field = next(f for f in fields if f.name == "version") + assert ver_field.type_expr == "IntegerType()" + + def test_is_land_is_boolean_type(self, fields: list[SchemaField]) -> None: + field = next(f for f in fields if f.name == "is_land") + assert field.type_expr == "BooleanType()" + + def test_country_is_string_type(self, fields: list[SchemaField]) -> None: + field = next(f for f in fields if f.name == "country") + assert field.type_expr == "StringType()" + + def test_admin_level_is_integer_type(self, fields: list[SchemaField]) -> None: + field = next(f for f in fields if f.name == "admin_level") + assert field.type_expr == "IntegerType()" + + def test_subtype_enum_is_string_type(self, fields: list[SchemaField]) -> None: + field = next(f for f in fields if f.name == "subtype") + assert field.type_expr == "StringType()" + + def test_theme_appears_once_at_model_position( + self, fields: list[SchemaField] + ) -> None: + theme_fields = [f for f in fields if f.name == "theme"] + assert len(theme_fields) == 1 + + def test_theme_and_type_present(self, fields: list[SchemaField]) -> None: + names = [f.name for f in fields] + assert "theme" in names + assert "type" in names + + +class _ColorA(Enum): + RED = "red" + GREEN = "green" + + +class _ColorB(Enum): + BLUE = "blue" + YELLOW = "yellow" + + +class _VariantA(BaseModel): + pass + + +class _VariantB(BaseModel): + pass + + +class TestUnionSchemaDeduplicate: + """build_schema deduplicates same-name fields from different union variants.""" + + @pytest.fixture + def fields(self) -> list[SchemaField]: + af_shared = AnnotatedField( + field_spec=FieldSpec( + name="id", + shape=Primitive(base_type="str"), + description=None, + is_required=True, + ), + variant_sources=None, + ) + af_color_a = AnnotatedField( + field_spec=FieldSpec( + name="color", + shape=Primitive(base_type="ColorA", source_type=_ColorA), + description=None, + is_required=True, + ), + variant_sources=(_VariantA,), + ) + af_color_b = AnnotatedField( + field_spec=FieldSpec( + name="color", + shape=Primitive(base_type="ColorB", source_type=_ColorB), + description=None, + is_required=True, + ), + variant_sources=(_VariantB,), + ) + spec = UnionSpec( + name="TestUnion", + description=None, + annotated_fields=[af_shared, af_color_a, af_color_b], + members=[], + discriminator_field=None, + discriminator_mapping=None, + source_annotation=object(), + common_base=BaseModel, + ) + return build_schema(spec) + + def test_one_schema_field_per_name(self, fields: list[SchemaField]) -> None: + color_fields = [f for f in fields if f.name == "color"] + assert len(color_fields) == 1 + + def test_color_field_is_string_type(self, fields: list[SchemaField]) -> None: + color_field = next(f for f in fields if f.name == "color") + assert color_field.type_expr == "StringType()" diff --git a/packages/overture-schema-codegen/tests/test_pyspark_test_renderer.py b/packages/overture-schema-codegen/tests/test_pyspark_test_renderer.py new file mode 100644 index 000000000..f167f26ba --- /dev/null +++ b/packages/overture-schema-codegen/tests/test_pyspark_test_renderer.py @@ -0,0 +1,1350 @@ +"""Tests for the generated conformance test module renderer.""" + +import ast +import re +from enum import Enum + +import pytest +from overture.schema.codegen.extraction.field import ArrayOf, ModelRef, Primitive +from overture.schema.codegen.extraction.specs import RecordSpec +from overture.schema.codegen.pyspark._primitive_fill import PRIMITIVE_FILL_TABLE +from overture.schema.codegen.pyspark.check_ir import ( + Check, + ColumnGuard, + ElementGuard, + ModelCheck, +) +from overture.schema.codegen.pyspark.constraint_dispatch import ( + ExpressionDescriptor, + ForbidIf, + MinFieldsSet, + RadioGroup, + RequireAnyOf, + RequireAnyTrue, + RequireIf, + _needs_explicit_fill, +) +from overture.schema.codegen.pyspark.renderer import render_model_module +from overture.schema.codegen.pyspark.test_data.base_row import ( + _primitive_default as _base_row_primitive_default, +) +from overture.schema.codegen.pyspark.test_renderer import ( + _fill_value_literal, +) +from overture.schema.codegen.pyspark.test_renderer import ( + render_test_module as _real_render_test_module, +) +from overture.schema.system.field_constraint.string import ( + CountryCodeAlpha2Constraint, + LanguageTagConstraint, + NoWhitespaceConstraint, + StrippedConstraint, +) +from overture.schema.system.field_path import ArrayPath, ScalarPath, parse +from overture.schema.system.model_constraint import FieldEqCondition, Not +from overture.schema.system.primitive.geom import GeometryType + +_path = parse + +# Placeholder expression import path -- tests parse the rendered source +# rather than executing it, so the import target need not be real. +_TEST_EXPRESSION_IMPORT = "_placeholder.expression_module" + +# Representative base_type for each SparkCategory in PRIMITIVE_FILL_TABLE. +_CATEGORY_BASE_TYPE: dict[str, str] = { + "int": "int32", + "float": "float64", + "bool": "bool", +} + + +def render_test_module(*args: object, **kwargs: object) -> str: + """Invoke the renderer with placeholder `expression_import`/`support_prefix`. + + Tests parse the rendered source rather than executing it, so neither + the expression import target nor the relative `_support` package depth + needs to match a real layout. Defining this as a free function (rather + than a fixture) keeps test bodies terse. + """ + kwargs.setdefault("expression_import", _TEST_EXPRESSION_IMPORT) + kwargs.setdefault("support_prefix", "..") + return _real_render_test_module(*args, **kwargs) # type: ignore[arg-type] + + +def make_check( + function: str, + target: object, + *, + args: tuple[object, ...] = (), + kwargs: tuple[tuple[str, object], ...] = (), + constraint_type: object = None, + label: str | None = None, + check_name: str | None = None, + guards: tuple[object, ...] = (), +) -> Check: + """Build a single-descriptor Check; defaults match Check/ExpressionDescriptor.""" + descriptor_kwargs: dict[str, object] = {"function": function} + if args: + descriptor_kwargs["args"] = args + if kwargs: + descriptor_kwargs["kwargs"] = kwargs + if constraint_type is not None: + descriptor_kwargs["constraint_type"] = constraint_type + if label is not None: + descriptor_kwargs["label"] = label + if check_name is not None: + descriptor_kwargs["check_name"] = check_name + return Check( + descriptors=(ExpressionDescriptor(**descriptor_kwargs),), # type: ignore[arg-type] + target=target, # type: ignore[arg-type] + guards=guards, # type: ignore[arg-type] + ) + + +def _array( + column: str, + inner_struct_paths: tuple[tuple[str, ...], ...] = (), + leaf_path: tuple[str, ...] = (), +) -> ArrayPath: + """Build an ArrayPath from a column name, inner struct paths, and a leaf path. + + Each entry in `inner_struct_paths` is `(prefix_structs..., inner_array_name)`: + the prefix names become struct segments and the last name becomes an + inner ArraySegment. + """ + column_path = _path(column) + if isinstance(column_path, ScalarPath): + prefix_structs = column_path.segments[:-1] + outer_name = column_path.segments[-1].name + prefix = ScalarPath(segments=prefix_structs) + path = prefix.append_array(outer_name, iter_count=1) + else: + assert isinstance(column_path, ArrayPath) # never a MapPath here + path = column_path + for sp in inner_struct_paths: + for n in sp[:-1]: + path = path.append_struct(n) + path = path.append_array(sp[-1], iter_count=1) + for n in leaf_path: + path = path.append_struct(n) + return path + + +class TestMapPathScenarios: + """MapPath field checks emit mutate_map_key / mutate_map_value scenarios.""" + + def test_map_key_emits_mutate_map_key(self) -> None: + check = make_check( + "check_pattern", + _path("names.common{key}"), + args=(r"^[a-z]+$",), + constraint_type=LanguageTagConstraint, + label="language tag", + ) + source = render_test_module("dictfeat", [check], []) + ast.parse(source) + assert "mutate_map_key(row, 'names.common', '123')" in source + assert "expected_field='names.common{key}'" in source + + def test_map_value_emits_mutate_map_value(self) -> None: + check = make_check( + "check_stripped", + _path("names{value}"), + constraint_type=StrippedConstraint, + ) + source = render_test_module("dictfeat", [check], []) + ast.parse(source) + assert "mutate_map_value(row, 'names', ' has spaces ')" in source + assert "expected_field='names{value}'" in source + + def test_map_mutation_helper_is_imported(self) -> None: + check = make_check( + "check_stripped", + _path("names{value}"), + constraint_type=StrippedConstraint, + ) + source = render_test_module("dictfeat", [check], []) + # Appears in both the import block and the scenario call. + assert source.count("mutate_map_value") >= 2 + + +class TestRenderTestModuleParseable: + def test_renders_valid_python_with_nodes(self) -> None: + nodes = [make_check("check_required", _path("country"))] + source = render_test_module("division_area", nodes, []) + ast.parse(source) + + def test_empty_nodes_renders_valid_python(self) -> None: + source = render_test_module("empty", [], []) + ast.parse(source) + + +class TestUnbuildableScenariosAreLoud: + """An unbuildable scenario must fail, not silently skip (false green).""" + + def test_skip_branch_fails_not_skips(self) -> None: + source = render_test_module("loud", [], []) + assert "validation_results.skipped" in source + assert "pytest.fail(" in source + assert "pytest.skip(" not in source + + +class TestBaseRow: + def test_default_base_rows_are_empty(self) -> None: + source = render_test_module("test", [], []) + assert "BASE_ROW_SPARSE: dict = {}" in source + assert "BASE_ROW_POPULATED: dict = {}" in source + + def test_provided_sparse_row_rendered(self) -> None: + source = render_test_module("test", [], [], base_row_sparse={"id": "abc"}) + assert "BASE_ROW_SPARSE: dict = " in source + assert "'id': 'abc'" in source + + def test_provided_populated_row_rendered(self) -> None: + source = render_test_module( + "test", + [], + [], + base_row_sparse={"id": "abc"}, + base_row_populated={"id": "abc", "names": {"primary": ""}}, + ) + assert "BASE_ROW_POPULATED: dict = " in source + assert "'names'" in source + + +class TestFieldScenarios: + def test_required_produces_none_value(self) -> None: + nodes = [make_check("check_required", _path("country"))] + source = render_test_module("test", nodes, []) + assert "Scenario(" in source + assert "set_at_path('country', None)" in source + assert "'country'" in source + assert "'required'" in source + + def test_enum_produces_invalid_string(self) -> None: + nodes = [ + make_check("check_enum", _path("subtype"), args=(["a", "b", "c"],)), + ] + source = render_test_module("test", nodes, []) + assert "__INVALID__" in source + assert "'enum'" in source + + def test_bounds_produces_out_of_range(self) -> None: + nodes = [ + make_check("check_bounds", _path("score"), kwargs=(("ge", 0.0),)), + ] + source = render_test_module("test", nodes, []) + assert "-1" in source or "-1.0" in source + assert "'bounds'" in source + + def test_bounds_preserves_int_type(self) -> None: + """Integer bound kwargs emit integer literals for IntegerType fields.""" + nodes = [ + make_check("check_bounds", _path("version"), kwargs=(("ge", 0),)), + ] + source = render_test_module("test", nodes, []) + assert "set_at_path('version', -1)" in source + + def test_bounds_preserves_float_type(self) -> None: + """Float bound kwargs emit float literals for DoubleType fields.""" + nodes = [ + make_check("check_bounds", _path("height"), kwargs=(("ge", 0.0),)), + ] + source = render_test_module("test", nodes, []) + assert "-1.0" in source + + def test_unknown_constraint_raises(self) -> None: + nodes = [make_check("check_something_unknown", _path("geom"))] + with pytest.raises(ValueError, match="Cannot render mutate expression"): + render_test_module("test", nodes, []) + + def test_pattern_without_constraint_type_raises(self) -> None: + """check_pattern with no constraint_type raises at codegen time.""" + nodes = [ + make_check("check_pattern", _path("wikidata.value"), args=(r"^Q\d+$",)), + ] + with pytest.raises(ValueError, match="Cannot render mutate expression"): + render_test_module("test", nodes, []) + + def test_no_whitespace_pattern_mutation_contains_whitespace(self) -> None: + """Mutation for NoWhitespaceConstraint must contain whitespace to violate ^\\S+$.""" + nodes = [ + make_check( + "check_pattern", + _path("id"), + args=(r"^\S+$",), + constraint_type=NoWhitespaceConstraint, + ), + ] + source = render_test_module("test", nodes, []) + match = re.search( + r"set_at_path\('id',\s*(.+?)\)", + source, + re.DOTALL, + ) + assert match, f"no id:pattern set_at_path found in:\n{source}" + mutation_value = match.group(1).strip() + assert re.search(r"\\s|\s", mutation_value.strip("'")), ( + f"mutation {mutation_value} does not contain whitespace" + ) + + def test_country_code_uses_invalid_value(self) -> None: + nodes = [ + make_check( + "check_pattern", + _path("country.value"), + constraint_type=CountryCodeAlpha2Constraint, + label="ISO 3166-1 alpha-2 country code", + check_name="country_code_alpha2", + ), + ] + source = render_test_module("test", nodes, []) + assert "'99'" in source + + def test_multiple_descriptors_produce_multiple_entries(self) -> None: + """A field with required + enum produces two scenario entries.""" + nodes = [ + Check( + descriptors=( + ExpressionDescriptor(function="check_required"), + ExpressionDescriptor(function="check_enum", args=(["a"],)), + ), + target=_path("subtype"), + ), + ] + source = render_test_module("test", nodes, []) + assert "'required'" in source + assert "'enum'" in source + + def test_min_length_produces_empty_list(self) -> None: + nodes = [ + make_check("check_array_min_length", _path("sources"), args=(1,)), + ] + source = render_test_module("test", nodes, []) + assert "set_at_path('sources', [])" in source + assert "expected_field='sources_min_length'" in source + + def test_max_length_produces_oversized_list(self) -> None: + nodes = [ + make_check("check_array_max_length", _path("connectors"), args=(3,)), + ] + source = render_test_module("test", nodes, []) + assert "[{}, {}, {}, {}]" in source or "[{}] * 4" in source + assert "expected_field='connectors_max_length'" in source + + def test_scenario_id_includes_model_name(self) -> None: + nodes = [make_check("check_required", _path("country"))] + source = render_test_module("division_area", nodes, []) + assert "division_area::country:required" in source + + def test_scenario_has_scaffold(self) -> None: + """Scenario includes a scaffold dict (empty when spec is None).""" + nodes = [make_check("check_required", _path("country"))] + source = render_test_module("test", nodes, []) + assert "scaffold={}" in source + + +class TestModelScenarios: + def test_radio_group_imports_mutation(self) -> None: + model_nodes = [ + ModelCheck( + descriptor=RadioGroup(field_names=("is_land", "is_territorial")), + ), + ] + source = render_test_module("test", [], model_nodes) + assert "mutate_radio_group" in source + assert "radio_group" in source + + def test_require_any_of_imports_mutation(self) -> None: + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("x", "y")), + ), + ] + source = render_test_module("test", [], model_nodes) + assert "mutate_require_any_of" in source + + def test_require_any_true_imports_mutation(self) -> None: + model_nodes = [ + ModelCheck( + descriptor=RequireAnyTrue( + conditions=( + FieldEqCondition("is_land", True), + FieldEqCondition("is_territorial", True), + ), + ), + ), + ] + source = render_test_module("test", [], model_nodes) + assert "mutate_require_any_true" in source + # Each `field == True` condition is disabled by setting the field False. + assert "'is_land': False" in source + assert "'is_territorial': False" in source + + def test_require_if_includes_condition(self) -> None: + model_nodes = [ + ModelCheck( + descriptor=RequireIf( + field_names=("admin_level",), + condition=FieldEqCondition("subtype", "country"), + ), + ), + ] + source = render_test_module("test", [], model_nodes) + assert "mutate_require_if" in source + assert "'country'" in source + + def test_model_scenario_uses_contains_assertion(self) -> None: + """Model-level tests use 'in' not '==' to check violation membership.""" + model_nodes = [ + ModelCheck( + descriptor=RadioGroup(field_names=("a", "b")), + ), + ] + source = render_test_module("test", [], model_nodes) + assert "assert expected in invalid_violations" in source + + def test_renders_valid_python(self) -> None: + model_nodes = [ + ModelCheck( + descriptor=RequireIf( + field_names=("admin_level",), + condition=FieldEqCondition("subtype", "country"), + ), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + + def test_enum_condition_value_renders_valid_python(self) -> None: + """Enum condition values must render as their string payload, not repr.""" + + class PlaceType(str, Enum): + COUNTY = "county" + + model_nodes = [ + ModelCheck( + descriptor=RequireIf( + field_names=("admin_level",), + condition=FieldEqCondition("subtype", PlaceType.COUNTY), + ), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert "'county'" in source + + def test_forbid_if_array_field_generates_fill_values(self) -> None: + """forbid_if targeting an array field emits fill_values with [{}].""" + model_nodes = [ + ModelCheck( + descriptor=ForbidIf( + field_names=("destinations",), + condition=FieldEqCondition("subtype", "road"), + field_shapes=( + ( + "destinations", + ArrayOf(element=Primitive(base_type="Destination")), + ), + ), + ), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert "fill_values" in source + assert "[{}]" in source + + def test_forbid_if_struct_field_generates_fill_values(self) -> None: + """forbid_if targeting a struct field emits fill_values with {}. + + Struct fields reach `_fill_value_literal` as `ModelRef` shapes, not + `Primitive` — `_needs_explicit_fill` only passes model references and + arrays for the `{}` / `[{}]` fill; string `Primitive`s are excluded. + """ + model_nodes = [ + ModelCheck( + descriptor=ForbidIf( + field_names=("road_surface",), + condition=FieldEqCondition("subtype", "road"), + field_shapes=( + ( + "road_surface", + ModelRef( + model=RecordSpec(name="RoadSurface", description=None) + ), + ), + ), + ), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert "fill_values" in source + assert "'road_surface': {}" in source + + def test_forbid_if_string_field_no_fill_values(self) -> None: + """forbid_if targeting a string field does not emit fill_values.""" + model_nodes = [ + ModelCheck( + descriptor=ForbidIf( + field_names=("class",), + condition=FieldEqCondition("subtype", "water"), + field_shapes=(), + ), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert "fill_values" not in source + + def test_forbid_if_not_condition_uses_negate(self) -> None: + """forbid_if with Not(FieldEqCondition) passes negate=True to mutation.""" + model_nodes = [ + ModelCheck( + descriptor=ForbidIf( + field_names=("destinations",), + condition=Not(FieldEqCondition("subtype", "road")), + field_shapes=(), + ), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert "negate=True" in source + assert "'road'" in source + + def test_require_any_of_nested_uses_array_path(self) -> None: + """require_any_of in an array element passes array_path to mutation.""" + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("labels", "symbols")), + target=_array("destinations"), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert 'array_path="destinations"' in source + + def test_require_any_of_nested_with_leaf_path(self) -> None: + """require_any_of nested in struct within array passes struct_path.""" + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("heading", "during")), + target=_array("access_restrictions", leaf_path=("when",)), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert 'array_path="access_restrictions"' in source + assert 'struct_path="when"' in source + + def test_require_any_of_top_level_no_array_path(self) -> None: + """Top-level require_any_of does not emit array_path.""" + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("a", "b")), + ), + ] + source = render_test_module("test", [], model_nodes) + assert "array_path" not in source + + def test_require_if_not_condition_uses_negate(self) -> None: + """require_if with Not(FieldEqCondition) passes negate=True to mutation.""" + model_nodes = [ + ModelCheck( + descriptor=RequireIf( + field_names=("class",), + condition=Not(FieldEqCondition("subtype", "road")), + ), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert "negate=True" in source + + def test_require_any_of_map_value_uses_map_path(self) -> None: + """require_any_of on a `dict[K, Model]` value model passes map_path.""" + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("foo", "bar")), + target=_path("subs{value}"), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert 'map_path="subs"' in source + assert "array_path" not in source + + def test_require_any_of_map_value_leaf_uses_struct_path(self) -> None: + """A struct-nested sub-model in a map value passes map_path + struct_path.""" + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("foo", "bar")), + target=_path("subs{value}.inner"), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert 'map_path="subs"' in source + assert 'struct_path="inner"' in source + + def test_min_fields_set_map_value_uses_map_path(self) -> None: + model_nodes = [ + ModelCheck( + descriptor=MinFieldsSet(field_names=("foo", "bar"), count=1), + target=_path("subs{value}"), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert 'map_path="subs"' in source + + def test_require_if_map_value_uses_map_path(self) -> None: + model_nodes = [ + ModelCheck( + descriptor=RequireIf( + field_names=("admin_level",), + condition=FieldEqCondition("subtype", "country"), + ), + target=_path("subs{value}"), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert 'map_path="subs"' in source + + def test_radio_group_map_value_raises(self) -> None: + """radio_group has no map-aware mutation; raise rather than emit a vacuous test.""" + model_nodes = [ + ModelCheck( + descriptor=RadioGroup(field_names=("a", "b")), + target=_path("subs{value}"), + ), + ] + with pytest.raises(ValueError, match="map_path"): + render_test_module("test", [], model_nodes) + + def test_require_any_of_map_key_projection_raises(self) -> None: + """A model can't be a dict key, so a KEY-projection model check is untestable.""" + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("foo", "bar")), + target=_path("subs{key}"), + ), + ] + with pytest.raises(ValueError, match="map key"): + render_test_module("test", [], model_nodes) + + def test_require_any_of_map_value_multi_segment_leaf_raises(self) -> None: + """The mutation struct_path is a single segment; a deeper leaf has no support.""" + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("foo", "bar")), + target=_path("subs{value}.a.b"), + ), + ] + with pytest.raises(ValueError, match="single segment"): + render_test_module("test", [], model_nodes) + + def test_model_scenario_uses_inline_lambda(self) -> None: + """Model scenarios emit mutate=lambda row: ... directly.""" + model_nodes = [ + ModelCheck( + descriptor=RadioGroup(field_names=("a", "b")), + ), + ] + source = render_test_module("test", [], model_nodes) + assert "mutate=lambda row:" in source + assert "mutate_radio_group(" in source + + def test_model_scenario_has_scaffold(self) -> None: + """Scenario includes a scaffold dict (empty when spec is None).""" + model_nodes = [ + ModelCheck( + descriptor=RadioGroup(field_names=("a", "b")), + ), + ] + source = render_test_module("test", [], model_nodes) + assert "Scenario(" in source + assert "scaffold={}" in source + + def test_min_fields_set_renders_mutation_call(self) -> None: + """MinFieldsSet dispatches to `mutate_min_fields_set`.""" + model_nodes = [ + ModelCheck( + descriptor=MinFieldsSet(field_names=("x", "y"), count=1), + ), + ] + source = render_test_module("test", [], model_nodes) + assert "mutate_min_fields_set(row, ['x', 'y'])" in source + import_match = re.search( + r"from \.\._support\.mutations\s+import\s+(.+?)(?:\n\n|\Z)", + source, + re.DOTALL, + ) + assert import_match is not None + assert "mutate_min_fields_set" in import_match.group(1) + + def test_require_any_of_with_inner_levels_raises(self) -> None: + """require_any_of does not accept inner_array_path.""" + model_nodes = [ + ModelCheck( + descriptor=RequireAnyOf(field_names=("a", "b")), + target=_array("outer", inner_struct_paths=(("inner",),)), + ), + ] + with pytest.raises(ValueError, match="inner_array_path"): + render_test_module("test", [], model_nodes) + + def test_radio_group_with_array_path_raises(self) -> None: + """radio_group takes no array kwargs; nodes with column_path raise.""" + model_nodes = [ + ModelCheck( + descriptor=RadioGroup(field_names=("a", "b")), + target=_array("outer"), + ), + ] + with pytest.raises(ValueError, match="array_path"): + render_test_module("test", [], model_nodes) + + def test_require_if_with_leaf_path_raises(self) -> None: + """require_if does not accept struct_path; nodes with leaf_path raise.""" + model_nodes = [ + ModelCheck( + descriptor=RequireIf( + field_names=("admin_level",), + condition=FieldEqCondition("subtype", "country"), + ), + target=_array("outer", leaf_path=("when",)), + ), + ] + with pytest.raises(ValueError, match="struct_path"): + render_test_module("test", [], model_nodes) + + def test_require_if_with_multi_inner_levels_raises(self) -> None: + """require_if only consumes one inner iteration; multi-level is rejected.""" + model_nodes = [ + ModelCheck( + descriptor=RequireIf( + field_names=("admin_level",), + condition=FieldEqCondition("subtype", "country"), + ), + target=_array("outer", inner_struct_paths=(("middle",), ("inner",))), + ), + ] + with pytest.raises(ValueError, match="multi-level inner struct paths"): + render_test_module("test", [], model_nodes) + + +class TestCrossArmModelCheckLabelCollision: + """Per-arm test labels must match the expression module's labels. + + The expression module is rendered once over the unfiltered model-check + list, so a cross-arm base-label collision earns a `_N` suffix there. A + per-arm test module must compute that suffix over the same unfiltered + list and filter rows afterward; computing it over the arm subset would + emit a bare `expected_field` the module never produces. + """ + + def test_per_arm_label_matches_module_label(self) -> None: + road = ModelCheck( + descriptor=RequireIf( + field_names=("class",), + condition=FieldEqCondition("subtype", "road"), + ), + arm="road", + ) + rail = ModelCheck( + descriptor=RequireIf( + field_names=("class",), + condition=FieldEqCondition("subtype", "rail"), + ), + arm="rail", + ) + model_checks = [road, rail] + + module = render_model_module("seg", [], model_checks, []) + module_labels = re.findall(r"field='(class_required[^']*)'", module) + road_label = module_labels[0] + + test_source = render_test_module("seg", [], model_checks, arm="road") + test_labels = re.findall(r"expected_field='(class_required[^']*)'", test_source) + + assert test_labels == [road_label], (test_labels, road_label) + + +class TestTestLayer: + @pytest.fixture(scope="class") + def empty_source(self) -> str: + return render_test_module("test", [], []) + + def test_test_scenario_sparse_present(self, empty_source: str) -> None: + assert "def test_scenario_sparse(" in empty_source + + def test_test_scenario_populated_present(self, empty_source: str) -> None: + assert "def test_scenario_populated(" in empty_source + + def test_test_baseline_sparse_present(self, empty_source: str) -> None: + assert "def test_baseline_sparse(" in empty_source + + def test_test_baseline_populated_present(self, empty_source: str) -> None: + assert "def test_baseline_populated(" in empty_source + + def test_sparse_results_fixture_present(self, empty_source: str) -> None: + assert "def sparse_results(" in empty_source + + def test_populated_results_fixture_present(self, empty_source: str) -> None: + assert "def populated_results(" in empty_source + + def test_assert_scenario_helper_present(self, empty_source: str) -> None: + assert "def _assert_scenario(" in empty_source + + def test_imports_scenario(self, empty_source: str) -> None: + assert "Scenario" in empty_source + + def test_uses_harness_imports(self, empty_source: str) -> None: + assert "from .._support.harness import" in empty_source + + def test_imports_set_at_path_only_when_field_scenarios_present(self) -> None: + # No field checks -> no set_at_path scenarios -> no import + empty = render_test_module("test", [], []) + assert "from .._support.helpers import set_at_path" not in empty + + # Field check -> set_at_path used -> import emitted + with_field = render_test_module( + "test", + [make_check("check_required", _path("country"))], + [], + ) + assert "from .._support.helpers import set_at_path" in with_field + + def test_scenario_checks_valid_and_invalid(self, empty_source: str) -> None: + assert "::valid" in empty_source + assert "::invalid" in empty_source + + def test_scenarios_list_type_annotation(self, empty_source: str) -> None: + assert "list[Scenario]" in empty_source + + def test_populated_tests_not_marked_skip(self, empty_source: str) -> None: + assert "pytest.mark.skip" not in empty_source + + +class TestStructUniqueCheckScenarios: + @pytest.fixture() + def sources_unique_output(self) -> str: + nodes = [make_check("check_struct_unique", _path("sources"))] + return render_test_module("test", nodes, []) + + def test_struct_unique_emits_scenario(self, sources_unique_output: str) -> None: + """struct_unique_check produces Scenario with scaffold and inline lambda.""" + assert "Scenario(" in sources_unique_output + assert "expected_field='sources_unique'" in sources_unique_output + assert "expected_check='struct_unique'" in sources_unique_output + + def test_struct_unique_imports_mutate_unique_items( + self, sources_unique_output: str + ) -> None: + assert ( + "from .._support.mutations import mutate_unique_items" + in sources_unique_output + ) + + def test_no_struct_unique_does_not_import_mutate_unique_items(self) -> None: + nodes = [make_check("check_required", _path("country"))] + source = render_test_module("test", nodes, []) + assert "mutate_unique_items" not in source + + def test_struct_unique_inline_lambda(self, sources_unique_output: str) -> None: + """struct_unique_check emits mutate=lambda row: mutate_unique_items(...).""" + assert "mutate=lambda row: mutate_unique_items(" in sources_unique_output + assert "'sources'" in sources_unique_output + + def test_struct_unique_nested_path_strips_suffix(self) -> None: + """Nested bracket path uses the structural field for mutation.""" + nodes = [ + make_check("check_struct_unique", _path("access_restrictions[].when.mode")), + ] + source = render_test_module("test", nodes, []) + # Black may wrap the long lambda — check parts separately + assert "mutate_unique_items(" in source + assert "'access_restrictions[].when.mode'" in source + assert "expected_field='access_restrictions[].when.mode_unique'" in source + + def test_struct_unique_renders_valid_python( + self, sources_unique_output: str + ) -> None: + ast.parse(sources_unique_output) + + def test_struct_unique_mixed_with_field_scenarios(self) -> None: + """struct_unique_check alongside normal field checks renders valid Python.""" + nodes = [ + make_check("check_required", _path("sources")), + make_check("check_struct_unique", _path("sources")), + ] + source = render_test_module("test", nodes, []) + ast.parse(source) + assert source.count("Scenario(") == 2 + + def test_struct_unique_has_scaffold(self, sources_unique_output: str) -> None: + """struct_unique_check Scenario includes scaffold dict.""" + assert "scaffold={}" in sources_unique_output + + +class TestArmFiltering: + """Per-arm test generation filters field checks by discriminator value.""" + + def _common_node(self) -> Check: + return make_check("check_required", _path("id")) + + def _road_node(self) -> Check: + return make_check( + "check_required", + _array("road_surface"), + guards=(ColumnGuard(discriminator="subtype", values=("road",)),), + ) + + def _rail_node(self) -> Check: + return make_check( + "check_required", + _array("rail_flags"), + guards=(ColumnGuard(discriminator="subtype", values=("rail",)),), + ) + + def _inner_disc_node(self) -> Check: + """Road-arm check with in-element discriminator (vehicle dimension).""" + return make_check( + "check_required", + _path("speed_limits[].when.vehicle[].value"), + guards=( + ColumnGuard(discriminator="subtype", values=("road",)), + ElementGuard(discriminator="dimension", values=("height", "length")), + ), + ) + + def test_arm_road_includes_common_and_road_checks(self) -> None: + nodes = [self._common_node(), self._road_node(), self._rail_node()] + source = render_test_module("test", nodes, [], arm="road") + assert "set_at_path('id'" in source + assert "road_surface" in source + assert "rail_flags" not in source + + def test_arm_rail_includes_common_and_rail_checks(self) -> None: + nodes = [self._common_node(), self._road_node(), self._rail_node()] + source = render_test_module("test", nodes, [], arm="rail") + assert "set_at_path('id'" in source + assert "rail_flags" in source + assert "road_surface" not in source + + def test_arm_includes_inner_disc_by_outer_variant(self) -> None: + """In-element discriminator checks emit when the outer Guard matches the arm.""" + nodes = [self._inner_disc_node()] + source = render_test_module("test", nodes, [], arm="road") + assert "vehicle" in source + + def test_arm_excludes_inner_disc_wrong_outer(self) -> None: + nodes = [self._inner_disc_node()] + source = render_test_module("test", nodes, [], arm="rail") + assert "vehicle" not in source + + def test_no_arm_includes_all_checks(self) -> None: + """Without arm filtering, all checks are included.""" + nodes = [self._common_node(), self._road_node(), self._rail_node()] + source = render_test_module("test", nodes, []) + assert "set_at_path('id'" in source + assert "road_surface" in source + assert "rail_flags" in source + + def test_arm_includes_model_checks(self) -> None: + """Arm-agnostic ModelChecks (arm=None) reach every arm test.""" + model_nodes = [ + ModelCheck( + descriptor=ForbidIf( + field_names=("rail_flags",), + condition=Not(FieldEqCondition("subtype", "rail")), + field_shapes=(), + ), + ), + ] + source = render_test_module("test", [], model_nodes, arm="road") + assert "mutate_forbid_if" in source + + def test_arm_excludes_other_arms_model_checks(self) -> None: + """A ModelCheck tagged for one arm does not appear in another arm's tests.""" + road_only = ModelCheck( + descriptor=RadioGroup(field_names=("road_flag_a", "road_flag_b")), + arm="road", + ) + road_source = render_test_module("test", [], [road_only], arm="road") + assert "mutate_radio_group" in road_source + rail_source = render_test_module("test", [], [road_only], arm="rail") + assert "mutate_radio_group" not in rail_source + + def test_arm_renders_valid_python(self) -> None: + nodes = [self._common_node(), self._road_node(), self._rail_node()] + source = render_test_module("test", nodes, [], arm="road") + ast.parse(source) + + def test_arm_filtering_ignores_inner_element_discriminator(self) -> None: + """Element guards on inner-union discriminators don't gate arm filtering. + + The inner `ElementGuard` discriminator (`dimension`) is unrelated + to the outer union arm (`subtype`). When an `ElementGuard` value + happens to coincide with an arm name, an `any(...)` filter would + wrongly include the check in that arm; the correct filter + consults only `ColumnGuard`s. + """ + check = make_check( + "check_required", + _path("speed_limits[].when.vehicle[].value"), + guards=( + ColumnGuard(discriminator="subtype", values=("road",)), + # ElementGuard values include "rail" by coincidence -- it's + # a vehicle dimension, not a segment subtype. Filtering by + # `any(...)` would let arm="rail" include the check. + ElementGuard(discriminator="dimension", values=("rail",)), + ), + ) + rail = render_test_module("test", [check], [], arm="rail") + assert "speed_limits" not in rail + road = render_test_module("test", [check], [], arm="road") + assert "speed_limits" in road + + +class TestFieldLabelCollisionSuffix: + """Colliding field-check `expected_field`s carry the suffix the module emits. + + The expression module is rendered once across every arm, so its + `(field, name)` collisions are defined over the full check list. The + per-arm test modules must derive `expected_field` from that same + full list -- not a post-arm-filter subset -- or they assert a field + the module never emits. + """ + + def _colliding_checks(self) -> list[Check]: + """Two `required` checks on one path, distinguished by inner union arm.""" + return [ + make_check( + "check_required", + _path("value"), + guards=( + ColumnGuard(discriminator="subtype", values=("road",)), + ElementGuard(discriminator="dimension", values=("axle_count",)), + ), + ), + make_check( + "check_required", + _path("value"), + guards=( + ColumnGuard(discriminator="subtype", values=("road",)), + ElementGuard(discriminator="dimension", values=("height",)), + ), + ), + ] + + def test_colliding_expected_fields_are_suffixed(self) -> None: + source = render_test_module("test", self._colliding_checks(), []) + ast.parse(source) + assert "expected_field='value_0'" in source + assert "expected_field='value_1'" in source + assert "expected_field='value'," not in source + + def test_suffix_survives_arm_filter(self) -> None: + """Both colliding checks share an arm; the per-arm file keeps both suffixes. + + Computing suffixes post-filter would still see the collision here + (both survive), so this alone is necessary but not sufficient -- + `test_suffix_computed_over_unfiltered_list` covers the case where + filtering would otherwise hide it. + """ + source = render_test_module("test", self._colliding_checks(), [], arm="road") + ast.parse(source) + assert "expected_field='value_0'" in source + assert "expected_field='value_1'" in source + + def test_suffix_computed_over_unfiltered_list(self) -> None: + """A surviving check keeps the suffix even when its collision sibling is filtered out. + + The two checks collide in the full list (both emit + `(value, required)`) but belong to different arms. The expression + module -- rendered across both arms -- emits `value_0` / `value_1`. + Each arm test sees only one of them after filtering; computing + the suffix from that one-element subset would wrongly drop it. + """ + checks = [ + make_check( + "check_required", + _path("value"), + guards=(ColumnGuard(discriminator="subtype", values=("road",)),), + ), + make_check( + "check_required", + _path("value"), + guards=(ColumnGuard(discriminator="subtype", values=("rail",)),), + ), + ] + road = render_test_module("test", checks, [], arm="road") + rail = render_test_module("test", checks, [], arm="rail") + ast.parse(road) + ast.parse(rail) + assert "expected_field='value_0'" in road + assert "expected_field='value_1'" in rail + # Neither arm asserts the bare, never-emitted label. + assert "expected_field='value'," not in road + assert "expected_field='value'," not in rail + + def test_noncolliding_field_check_stays_bare(self) -> None: + nodes = [ + make_check("check_required", _path("value")), + make_check("check_bounds", _path("value"), kwargs=(("ge", 0),)), + ] + source = render_test_module("test", nodes, []) + ast.parse(source) + assert "expected_field='value'," in source + assert "expected_field='value_0'" not in source + + +class TestForbidIfNonStringFillValues: + """fill_values for non-string scalar ForbidIf fields must be typed literals.""" + + def test_forbid_if_int_field_generates_int_fill_value(self) -> None: + """forbid_if targeting an int field emits fill_values with 0, not {}.""" + model_nodes = [ + ModelCheck( + descriptor=ForbidIf( + field_names=("version",), + condition=FieldEqCondition("subtype", "road"), + field_shapes=(("version", Primitive(base_type="int32")),), + ), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert "fill_values" in source + assert "'version': 0" in source + assert "'version': {}" not in source + + def test_forbid_if_bool_field_generates_bool_fill_value(self) -> None: + """forbid_if targeting a bool field emits fill_values with False, not {}.""" + model_nodes = [ + ModelCheck( + descriptor=ForbidIf( + field_names=("flag",), + condition=FieldEqCondition("subtype", "road"), + field_shapes=(("flag", Primitive(base_type="bool")),), + ), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert "fill_values" in source + assert "'flag': False" in source + assert "'flag': {}" not in source + + def test_forbid_if_float_field_generates_float_fill_value(self) -> None: + """forbid_if targeting a float field emits fill_values with 0.0, not {}.""" + model_nodes = [ + ModelCheck( + descriptor=ForbidIf( + field_names=("score",), + condition=FieldEqCondition("subtype", "road"), + field_shapes=(("score", Primitive(base_type="float64")),), + ), + ), + ] + source = render_test_module("test", [], model_nodes) + ast.parse(source) + assert "fill_values" in source + assert "'score': 0.0" in source + assert "'score': {}" not in source + + def test_string_primitive_in_field_shapes_raises(self) -> None: + """_fill_value_literal raises ValueError if a string-typed Primitive reaches it. + + String primitives must not appear in field_shapes (the contract is that + `_needs_explicit_fill` filters them out). A direct violation raises loudly + instead of silently emitting `{}`. + """ + model_nodes = [ + ModelCheck( + descriptor=ForbidIf( + field_names=("label",), + condition=FieldEqCondition("subtype", "road"), + field_shapes=(("label", Primitive(base_type="str")),), + ), + ), + ] + with pytest.raises(ValueError, match="unhandled Primitive base_type"): + render_test_module("test", [], model_nodes) + + +class TestFillValueLiteralOtherCategory: + """_fill_value_literal raises for Primitive base types in category 'other'. + + 'Geometry' maps to `primitive_spark_category` -> 'other'. A ForbidIf + field_shapes entry containing such a shape must raise at generation time + rather than silently emitting `{}` (a struct literal) for a binary column. + """ + + def test_geometry_primitive_raises_directly(self) -> None: + """_fill_value_literal raises ValueError for a Primitive of category 'other'. + + Calls `_fill_value_literal` directly with a `Geometry` Primitive + (category 'other') to confirm the raise is unconditional rather + than guarded by a registry lookup. + """ + with pytest.raises(ValueError, match="unhandled Primitive base_type"): + _fill_value_literal(Primitive(base_type="Geometry")) + + def test_geometry_primitive_in_field_shapes_raises(self) -> None: + """_fill_value_literal raises ValueError for a Geometry-typed Primitive. + + 'Geometry' is category 'other' in `primitive_spark_category`. Without + the fix, the 'other' branch falls through to the struct `return "{}"`, + silently emitting an invalid fill value for a binary column. + """ + model_nodes = [ + ModelCheck( + descriptor=ForbidIf( + field_names=("geometry",), + condition=FieldEqCondition("subtype", "road"), + field_shapes=(("geometry", Primitive(base_type="Geometry")),), + ), + ), + ] + with pytest.raises(ValueError, match="unhandled Primitive base_type"): + render_test_module("test", [], model_nodes) + + +class TestLinearRangeMutations: + @pytest.mark.parametrize( + ("function", "expected_value"), + [ + ("check_linear_range_length", "[0.5]"), + ("check_linear_range_bounds", "[1.5, 2.0]"), + ("check_linear_range_order", "[0.8, 0.2]"), + ], + ) + def test_mutation_renders(self, function: str, expected_value: str) -> None: + nodes = [make_check(function, _path("between"))] + source = render_test_module("test", nodes, []) + assert expected_value in source + + +class TestGeometryTypeMutations: + def test_point_allowed_emits_linestring(self) -> None: + """When Point is allowed, inject LineString as the wrong type.""" + nodes = [ + make_check( + "check_geometry_type", + _path("geometry"), + args=(GeometryType.POINT,), + ), + ] + source = render_test_module("test", nodes, []) + assert "LineString" in source or "LINESTRING" in source + + def test_polygon_allowed_emits_point(self) -> None: + """When Point is not allowed, inject Point as the wrong type.""" + nodes = [ + make_check( + "check_geometry_type", + _path("geometry"), + args=(GeometryType.POLYGON, GeometryType.MULTI_POLYGON), + ), + ] + source = render_test_module("test", nodes, []) + assert "POINT" in source or "Point" in source + + def test_geometry_type_renders_valid_python(self) -> None: + nodes = [ + make_check( + "check_geometry_type", + _path("geometry"), + args=(GeometryType.POINT,), + ), + ] + source = render_test_module("test", nodes, []) + ast.parse(source) + + def test_geometry_type_uses_wkt_strings(self) -> None: + """Geometry scenarios use WKT strings, not shapely constructor calls.""" + nodes = [ + make_check( + "check_geometry_type", + _path("geometry"), + args=(GeometryType.POINT,), + ), + ] + source = render_test_module("test", nodes, []) + assert "shapely" not in source + assert "LINESTRING" in source or "LineString" in source + + def test_all_candidates_allowed_raises(self) -> None: + """When all geometry candidates are allowed, scenario generation raises.""" + nodes = [ + make_check( + "check_geometry_type", + _path("geometry"), + args=( + GeometryType.POINT, + GeometryType.LINE_STRING, + GeometryType.GEOMETRY_COLLECTION, + ), + ), + ] + with pytest.raises(ValueError, match="Cannot render mutate expression"): + render_test_module("test", nodes, []) + + def test_no_geometry_type_no_shapely_imports(self) -> None: + """Shapely imports are absent when no geometry type scenario exists.""" + nodes = [make_check("check_required", _path("country"))] + source = render_test_module("test", nodes, []) + assert "shapely" not in source + + +class TestPrimitiveFillTableConsistency: + """The shared PRIMITIVE_FILL_TABLE drives all three fill-related functions. + + Every category in the table must be accepted by `_needs_explicit_fill`, + produce a non-raising `_fill_value_literal`, and yield the matching + `_primitive_default` runtime value. A future category added to the table + but not wired into a consumer will fail here. + """ + + def test_category_base_type_covers_table(self) -> None: + """_CATEGORY_BASE_TYPE must cover every key in PRIMITIVE_FILL_TABLE. + + If a category is added to the table without a representative base_type, + the other consistency tests would raise KeyError with a misleading trace + rather than a clear assertion. This test catches the gap loudly. + """ + assert set(_CATEGORY_BASE_TYPE) == set(PRIMITIVE_FILL_TABLE), ( + "Add a representative base_type to _CATEGORY_BASE_TYPE for each " + "new PRIMITIVE_FILL_TABLE key (and remove entries for deleted keys)." + ) + + def test_table_covers_needs_explicit_fill(self) -> None: + for category in PRIMITIVE_FILL_TABLE: + shape = Primitive(base_type=_CATEGORY_BASE_TYPE[category]) + assert _needs_explicit_fill(shape), ( + f"category {category!r} not accepted by _needs_explicit_fill" + ) + + def test_table_covers_fill_value_literal(self) -> None: + for category, (literal, _) in PRIMITIVE_FILL_TABLE.items(): + shape = Primitive(base_type=_CATEGORY_BASE_TYPE[category]) + assert _fill_value_literal(shape) == literal, ( + f"category {category!r} literal mismatch" + ) + + def test_table_covers_primitive_default(self) -> None: + for category, (_, runtime_value) in PRIMITIVE_FILL_TABLE.items(): + base_type = _CATEGORY_BASE_TYPE[category] + result = _base_row_primitive_default(base_type) + assert result == runtime_value, ( + f"category {category!r} runtime value mismatch" + ) + assert type(result) is type(runtime_value), ( + f"category {category!r} runtime type mismatch" + ) diff --git a/packages/overture-schema-codegen/tests/test_reverse_references.py b/packages/overture-schema-codegen/tests/test_reverse_references.py index fb8e1e41a..02d7ffcc2 100644 --- a/packages/overture-schema-codegen/tests/test_reverse_references.py +++ b/packages/overture-schema-codegen/tests/test_reverse_references.py @@ -14,14 +14,21 @@ has_name, lookup_by_name, make_union_spec, + spec_for_model, ) from overture.schema.codegen.extraction.enum_extraction import extract_enum -from overture.schema.codegen.extraction.model_extraction import ( - expand_model_tree, - extract_model, +from overture.schema.codegen.extraction.field import ( + ArrayOf, + ConstraintSource, + Primitive, ) from overture.schema.codegen.extraction.newtype_extraction import extract_newtype -from overture.schema.codegen.extraction.specs import PydanticTypeSpec, TypeIdentity +from overture.schema.codegen.extraction.specs import ( + NewTypeSpec, + PydanticTypeSpec, + RecordSpec, + TypeIdentity, +) from overture.schema.codegen.layout.type_collection import ( collect_all_supplementary_types, ) @@ -49,13 +56,12 @@ def test_model_referencing_type_produces_used_by_entry( target_name: str, ) -> None: """Model referencing a type produces a 'used by' entry on that type.""" - model_spec = extract_model(model_class, entry_point=model_name) - expand_model_tree(model_spec) - all_specs = collect_all_supplementary_types([model_spec]) + expanded = spec_for_model(model_class, entry_point=model_name) + all_specs = collect_all_supplementary_types([expanded]) assert has_name(all_specs, target_name) - result = compute_reverse_references([model_spec], all_specs) + result = compute_reverse_references([expanded], all_specs) entries = lookup_by_name(result, target_name) assert len(entries) == 1 @@ -84,6 +90,41 @@ def test_newtype_inheriting_from_newtype_produces_used_by_entry() -> None: assert entries[0].kind == UsedByKind.NEWTYPE +def test_newtype_inheriting_through_array_layer_produces_used_by_entry() -> None: + """A NewType chaining through an array NewType inherits the inner + NewType's provenance from the array layer, not just the terminal scalar.""" + Inner = NewType("Inner", str) + Outer = NewType("Outer", list) + + outer_spec = NewTypeSpec( + name="Outer", + description=None, + shape=ArrayOf( + element=Primitive(base_type="str"), + constraints=( + ConstraintSource( + source_ref=Inner, source_name="Inner", constraint=object() + ), + ), + ), + ) + inner_spec = NewTypeSpec( + name="Inner", description=None, shape=Primitive(base_type="str") + ) + + all_specs = { + TypeIdentity(Outer, "Outer"): outer_spec, + TypeIdentity(Inner, "Inner"): inner_spec, + } + + result = compute_reverse_references([], all_specs) + + entries = lookup_by_name(result, "Inner") + assert len(entries) == 1 + assert entries[0].identity.name == "Outer" + assert entries[0].kind == UsedByKind.NEWTYPE + + def test_union_members_have_used_by_entries() -> None: """Union members have 'used by' entries pointing to the union feature.""" # Create a union spec with RoadSegment as a member @@ -95,8 +136,8 @@ def test_union_members_have_used_by_entries() -> None: ) # Extract the member - road_spec = extract_model(RoadSegment) - expand_model_tree(road_spec) + road_spec = spec_for_model(RoadSegment) + assert isinstance(road_spec, RecordSpec) all_specs = {TypeIdentity(RoadSegment, "RoadSegment"): road_spec} result = compute_reverse_references([union_spec], all_specs) @@ -109,8 +150,8 @@ def test_union_members_have_used_by_entries() -> None: def test_self_references_filtered_out() -> None: """Self-references are filtered out (handles recursive types).""" - tree_spec = extract_model(TreeNode, entry_point="TreeNode") - expand_model_tree(tree_spec) + tree_spec = spec_for_model(TreeNode, entry_point="TreeNode") + assert isinstance(tree_spec, RecordSpec) # Manually add TreeNode to all_specs to test self-reference filtering all_specs = {TypeIdentity(TreeNode, "TreeNode"): tree_spec} @@ -124,10 +165,8 @@ def test_self_references_filtered_out() -> None: def test_deduplication_same_type_multiple_fields() -> None: """Deduplication works when same type is referenced via multiple fields.""" - instrument_spec = extract_model(Instrument, entry_point="Instrument") - venue_spec = extract_model(Venue, entry_point="Venue") - expand_model_tree(instrument_spec) - expand_model_tree(venue_spec) + instrument_spec = spec_for_model(Instrument, entry_point="Instrument") + venue_spec = spec_for_model(Venue, entry_point="Venue") all_specs = collect_all_supplementary_types([instrument_spec, venue_spec]) assert has_name(all_specs, "Id") @@ -145,14 +184,13 @@ def test_deduplication_same_type_multiple_fields() -> None: def test_pydantic_type_has_used_by_from_feature() -> None: """Pydantic type in all_specs gets used-by entries from features referencing it.""" - model_spec = extract_model(FeatureWithUrl, entry_point="FeatureWithUrl") - expand_model_tree(model_spec) - all_specs = collect_all_supplementary_types([model_spec]) + expanded = spec_for_model(FeatureWithUrl, entry_point="FeatureWithUrl") + all_specs = collect_all_supplementary_types([expanded]) assert has_name(all_specs, "HttpUrl") assert isinstance(lookup_by_name(all_specs, "HttpUrl"), PydanticTypeSpec) - result = compute_reverse_references([model_spec], all_specs) + result = compute_reverse_references([expanded], all_specs) entries = lookup_by_name(result, "HttpUrl") assert any(e.identity.name == "FeatureWithUrl" for e in entries) @@ -176,10 +214,8 @@ class FeatureBeta(BaseModel): FeatureBeta.__name__ = "Feature" FeatureBeta.__module__ = "beta.models" - spec_a = extract_model(FeatureAlpha, entry_point="Feature") - spec_b = extract_model(FeatureBeta, entry_point="Feature") - expand_model_tree(spec_a) - expand_model_tree(spec_b) + spec_a = spec_for_model(FeatureAlpha, entry_point="Feature") + spec_b = spec_for_model(FeatureBeta, entry_point="Feature") enum_id = TypeIdentity(SharedEnum, "SharedEnum") all_specs = {enum_id: extract_enum(SharedEnum)} @@ -201,10 +237,8 @@ def test_sorting_models_before_newtypes() -> None: # Create a synthetic NewType that wraps Id CustomId = NewType("CustomId", Id) - instrument_spec = extract_model(Instrument, entry_point="Instrument") - venue_spec = extract_model(Venue, entry_point="Venue") - expand_model_tree(instrument_spec) - expand_model_tree(venue_spec) + instrument_spec = spec_for_model(Instrument, entry_point="Instrument") + venue_spec = spec_for_model(Venue, entry_point="Venue") all_specs = collect_all_supplementary_types([instrument_spec, venue_spec]) # Add the CustomId NewType which references Id diff --git a/packages/overture-schema-codegen/tests/test_specs.py b/packages/overture-schema-codegen/tests/test_specs.py index 0780e2fda..51da3a20a 100644 --- a/packages/overture-schema-codegen/tests/test_specs.py +++ b/packages/overture-schema-codegen/tests/test_specs.py @@ -13,222 +13,94 @@ from overture.schema.codegen.extraction.specs import ( AnnotatedField, EnumSpec, - FeatureSpec, FieldSpec, ModelSpec, NewTypeSpec, + RecordSpec, TypeIdentity, is_union_alias, ) -from overture.schema.codegen.extraction.type_analyzer import TypeInfo, TypeKind +from overture.schema.system.primitive import int32 from pydantic import BaseModel, Field -class TestFeatureSpecProtocol: - """Tests for FeatureSpec protocol compliance.""" - - def test_model_spec_satisfies_feature_spec(self) -> None: - """ModelSpec satisfies the FeatureSpec protocol.""" - +class TestModelSpec: + def test_record_spec_is_model_spec(self) -> None: class Simple(BaseModel): name: str - spec = extract_model(Simple) - # Protocol compliance check - assert isinstance(spec, FeatureSpec) - # Verify protocol attributes + spec: ModelSpec = extract_model(Simple) + assert isinstance(spec, RecordSpec) assert spec.name == "Simple" assert isinstance(spec.fields, list) assert spec.source_type is Simple class TestFieldSpec: - """Tests for FieldSpec dataclass.""" - - def test_fieldspec_stores_basic_attributes(self) -> None: - """FieldSpec should store name, type_info, description, is_required.""" - field_spec = FieldSpec( - name="test_field", - type_info=STR_TYPE, - description="A test field", - is_required=True, - ) - - assert field_spec.name == "test_field" - assert field_spec.type_info == STR_TYPE - assert field_spec.description == "A test field" - assert field_spec.is_required is True - - def test_fieldspec_optional_field(self) -> None: - """FieldSpec should handle optional fields.""" - optional_str = TypeInfo( - base_type="str", kind=TypeKind.PRIMITIVE, is_optional=True - ) - - field_spec = FieldSpec( + def test_carries_shape_and_optional_flag(self) -> None: + fs = FieldSpec( name="optional_field", - type_info=optional_str, + shape=STR_TYPE, description=None, is_required=False, + is_optional=True, ) - - assert field_spec.is_required is False - assert field_spec.description is None - - -class TestModelSpec: - """Tests for ModelSpec dataclass.""" - - def test_modelspec_stores_basic_attributes(self) -> None: - """ModelSpec should store name, description, fields.""" - field = FieldSpec( - name="id", - type_info=STR_TYPE, - description="Unique identifier", - is_required=True, - ) - - model_spec = ModelSpec( - name="TestModel", - description="A test model", - fields=[field], - ) - - assert model_spec.name == "TestModel" - assert model_spec.description == "A test model" - assert len(model_spec.fields) == 1 - assert model_spec.fields[0].name == "id" - - def test_entry_point_defaults_to_none(self) -> None: - spec = ModelSpec(name="M", description=None) - assert spec.entry_point is None + assert fs.name == "optional_field" + assert fs.shape is STR_TYPE + assert fs.is_required is False + assert fs.is_optional is True class TestAnnotatedField: - """Tests for AnnotatedField wrapper.""" - def test_stores_field_and_variant_sources(self) -> None: - """AnnotatedField pairs a FieldSpec with variant provenance.""" - fs = FieldSpec(name="x", type_info=STR_TYPE, description=None, is_required=True) - af = AnnotatedField(field_spec=fs, variant_sources=("RoadSegment",)) + class RoadSegment(BaseModel): + pass + + fs = FieldSpec(name="x", shape=STR_TYPE) + af = AnnotatedField(field_spec=fs, variant_sources=(RoadSegment,)) assert af.field_spec is fs - assert af.variant_sources == ("RoadSegment",) + assert af.variant_sources == (RoadSegment,) def test_none_variant_sources_means_shared(self) -> None: - """variant_sources=None indicates a shared field.""" - fs = FieldSpec(name="x", type_info=STR_TYPE, description=None, is_required=True) + fs = FieldSpec(name="x", shape=STR_TYPE) af = AnnotatedField(field_spec=fs, variant_sources=None) assert af.variant_sources is None -class TestFieldSpecModelTree: - """Tests for FieldSpec model and starts_cycle fields.""" - - def test_model_defaults_to_none(self) -> None: - field_spec = FieldSpec( - name="test", type_info=STR_TYPE, description=None, is_required=True - ) - assert field_spec.model is None - - def test_starts_cycle_defaults_to_false(self) -> None: - field_spec = FieldSpec( - name="test", type_info=STR_TYPE, description=None, is_required=True - ) - assert field_spec.starts_cycle is False - - def test_model_can_hold_model_spec(self) -> None: - type_info = TypeInfo(base_type="Address", kind=TypeKind.MODEL) - sub = ModelSpec(name="Address", description=None) - field_spec = FieldSpec( - name="address", - type_info=type_info, - description=None, - is_required=True, - model=sub, - ) - assert field_spec.model is sub - - def test_starts_cycle_can_be_set(self) -> None: - type_info = TypeInfo(base_type="Node", kind=TypeKind.MODEL) - sub = ModelSpec(name="Node", description=None) - field_spec = FieldSpec( - name="parent", - type_info=type_info, - description=None, - is_required=False, - model=sub, - starts_cycle=True, - ) - assert field_spec.starts_cycle is True - assert field_spec.model is sub - - def test_starts_cycle_without_model_is_nonsensical(self) -> None: - """starts_cycle=True with model=None is expressible but invalid. - - expand_model_tree never produces this combination -- starts_cycle - is only set when model points to the cycle-causing ModelSpec. - Document the invariant so violations stand out. - """ - type_info = TypeInfo(base_type="Node", kind=TypeKind.MODEL) - field_spec = FieldSpec( - name="parent", - type_info=type_info, - description=None, - is_required=False, - starts_cycle=True, - ) - # Expressible but meaningless: cycle to nowhere - assert field_spec.starts_cycle is True - assert field_spec.model is None - - class TestIsUnionAlias: - """Tests for is_union_alias predicate.""" - def test_annotated_union_of_models_returns_true(self) -> None: - """Annotated[Union of BaseModels] is a union alias.""" - class A(BaseModel): x: int class B(BaseModel): y: str - union_type = Annotated[A | B, Field(description="test")] - assert is_union_alias(union_type) is True + assert is_union_alias(Annotated[A | B, Field(description="test")]) is True def test_model_class_returns_false(self) -> None: - """A concrete BaseModel class is not a union alias.""" - class A(BaseModel): x: int assert is_union_alias(A) is False def test_plain_string_returns_false(self) -> None: - """A plain string is not a union alias.""" assert is_union_alias("not a type") is False def test_non_model_union_returns_false(self) -> None: - """A union of non-model types is not a union alias.""" assert is_union_alias(str | int) is False class TestUnionSpec: - """Tests for UnionSpec data structure.""" - def test_fields_property_returns_plain_field_specs(self) -> None: - """UnionSpec.fields property returns list[FieldSpec] from annotated_fields.""" - fs1 = FieldSpec( - name="a", type_info=STR_TYPE, description=None, is_required=True - ) - fs2 = FieldSpec( - name="b", type_info=STR_TYPE, description=None, is_required=False - ) + class X(BaseModel): + pass + + fs1 = FieldSpec(name="a", shape=STR_TYPE) + fs2 = FieldSpec(name="b", shape=STR_TYPE, is_required=False) spec = make_union_spec( annotated_fields=[ AnnotatedField(field_spec=fs1, variant_sources=None), - AnnotatedField(field_spec=fs2, variant_sources=("X",)), + AnnotatedField(field_spec=fs2, variant_sources=(X,)), ], ) assert spec.fields == [fs1, fs2] @@ -240,20 +112,13 @@ def test_frozen(self) -> None: with pytest.raises(AttributeError): ti.obj = str # type: ignore[misc] - def test_same_obj_equal(self) -> None: + def test_equality_by_obj_identity(self) -> None: a = TypeIdentity(obj=int, name="int") b = TypeIdentity(obj=int, name="integer") + c = TypeIdentity(obj=str, name="int") assert a == b - - def test_same_obj_same_hash(self) -> None: - a = TypeIdentity(obj=int, name="int") - b = TypeIdentity(obj=int, name="integer") assert hash(a) == hash(b) - - def test_different_obj_not_equal(self) -> None: - a = TypeIdentity(obj=int, name="int") - b = TypeIdentity(obj=str, name="int") - assert a != b + assert a != c def test_works_as_dict_key(self) -> None: ti = TypeIdentity(obj=int, name="int") @@ -269,37 +134,23 @@ def test_not_equal_to_non_identity(self) -> None: class TestSpecIdentity: - def test_model_spec_identity(self) -> None: - spec = ModelSpec(name="Foo", description=None, source_type=SimpleModel) - ident = spec.identity - assert isinstance(ident, TypeIdentity) - assert ident.obj is SimpleModel - assert ident.name == "Foo" + def test_record_spec_identity(self) -> None: + spec = RecordSpec(name="Foo", description=None, source_type=SimpleModel) + assert spec.identity.obj is SimpleModel + assert spec.identity.name == "Foo" def test_enum_spec_identity(self) -> None: spec = EnumSpec(name="Color", description=None, source_type=InstrumentFamily) - ident = spec.identity - assert ident.obj is InstrumentFamily - assert ident.name == "Color" + assert spec.identity.obj is InstrumentFamily def test_newtype_spec_identity(self) -> None: - from overture.schema.system.primitive import int32 - spec = NewTypeSpec( - name="int32", description=None, type_info=STR_TYPE, source_type=int32 + name="int32", description=None, shape=STR_TYPE, source_type=int32 ) - ident = spec.identity - assert ident.obj is int32 - assert ident.name == "int32" + assert spec.identity.obj is int32 def test_union_spec_identity(self) -> None: sentinel = object() spec = make_union_spec("TestUnion", source_annotation=sentinel) - ident = spec.identity - assert ident.obj is sentinel - assert ident.name == "TestUnion" - - def test_model_spec_satisfies_feature_protocol_with_identity(self) -> None: - spec = ModelSpec(name="Foo", description=None, source_type=SimpleModel) - feature: FeatureSpec = spec - assert feature.identity.obj is SimpleModel + assert spec.identity.obj is sentinel + assert spec.identity.name == "TestUnion" diff --git a/packages/overture-schema-codegen/tests/test_type_analyzer.py b/packages/overture-schema-codegen/tests/test_type_analyzer.py index bbf8373fd..144214b75 100644 --- a/packages/overture-schema-codegen/tests/test_type_analyzer.py +++ b/packages/overture-schema-codegen/tests/test_type_analyzer.py @@ -1,18 +1,44 @@ -"""Tests for type analysis.""" +"""Tests for `analyze_type`: annotation -> `FieldShape` analysis.""" from enum import Enum from typing import Annotated, Any, Literal, NewType, Optional import pytest -from annotated_types import Ge +from annotated_types import Ge, MaxLen, MinLen +from overture.schema.codegen.extraction.field import ( + AnyScalar, + ArrayOf, + ConstraintSource, + FieldShape, + LiteralScalar, + MapOf, + ModelRef, + NewTypeShape, + Primitive, + UnionRef, +) +from overture.schema.codegen.extraction.field_walk import ( + all_constraints, + list_depth, +) +from overture.schema.codegen.extraction.length_constraints import ( + ArrayMinLen, + ScalarMinLen, +) +from overture.schema.codegen.extraction.literal_alternatives import ( + LiteralAlternatives, +) +from overture.schema.codegen.extraction.specs import RecordSpec, UnionSpec from overture.schema.codegen.extraction.type_analyzer import ( - TypeInfo, - TypeKind, + UnresolvedForwardRefError, UnsupportedUnionError, analyze_type, + attach_constraints, single_literal_value, + unwrap_list, ) -from overture.schema.system.primitive import float64, int32 +from overture.schema.common.scoping.vehicle import VehicleSelector +from overture.schema.system.primitive import int32 from overture.schema.system.ref import Id from overture.schema.system.string import ( HexColor, @@ -24,570 +50,441 @@ from typing_extensions import Sentinel -@pytest.fixture() -def id_type_info() -> TypeInfo: - return analyze_type(Id) +def _shape(annotation: object) -> FieldShape: + shape, _, _ = analyze_type(annotation) + return shape -@pytest.fixture() -def hex_color_type_info() -> TypeInfo: - return analyze_type(HexColor) +def _is_optional(annotation: object) -> bool: + _, is_optional, _ = analyze_type(annotation) + return is_optional -class TestAnalyzeTypePrimitives: - """Tests for primitive type analysis.""" +def _description(annotation: object) -> str | None: + _, _, description = analyze_type(annotation) + return description - @pytest.mark.parametrize("annotation", [str, int, float, bool]) - def test_builtin_returns_primitive_type_info(self, annotation: type) -> None: - """Builtin type annotations return PRIMITIVE TypeInfo with matching base_type.""" - result = analyze_type(annotation) - assert result.base_type == annotation.__name__ - assert result.kind == TypeKind.PRIMITIVE - assert result.is_optional is False - assert result.is_list is False +class TestPrimitives: + @pytest.mark.parametrize("annotation", [str, int, float, bool]) + def test_builtin_emits_primitive(self, annotation: type) -> None: + shape = _shape(annotation) + assert isinstance(shape, Primitive) + assert shape.base_type == annotation.__name__ + assert shape.source_type is annotation + def test_any_emits_any_scalar(self) -> None: + shape = _shape(Any) + assert isinstance(shape, AnyScalar) -class TestAnalyzeTypeSentinel: - """Tests for Sentinel type filtering in unions. - Pydantic uses `typing_extensions.Sentinel` instances (like ``) - in union types for optional fields. The type analyzer filters these out - alongside `None` when processing unions. - """ +class TestSentinel: + """`Sentinel` arms in unions are filtered alongside `None`.""" @pytest.fixture() - def missing_sentinel(self) -> object: + def missing(self) -> object: return Sentinel("MISSING") - def test_sentinel_filtered_from_union(self, missing_sentinel: object) -> None: - """Sentinel is filtered out, leaving the concrete type.""" - result = analyze_type(str | missing_sentinel) # type: ignore[arg-type] - - assert result.base_type == "str" - assert result.kind == TypeKind.PRIMITIVE - assert result.is_optional is False - - def test_sentinel_with_none_sets_optional(self, missing_sentinel: object) -> None: - """Sentinel + None both filtered; None triggers is_optional.""" - result = analyze_type(str | missing_sentinel | None) # type: ignore[arg-type] - - assert result.base_type == "str" - assert result.kind == TypeKind.PRIMITIVE - assert result.is_optional is True - - -class TestAnalyzeTypeOptional: - """Tests for Optional type analysis.""" - - def test_pipe_none_sets_is_optional(self) -> None: - """str | None returns TypeInfo with is_optional=True.""" - result = analyze_type(str | None) - - assert result.base_type == "str" - assert result.kind == TypeKind.PRIMITIVE - assert result.is_optional is True - assert result.is_list is False - - def test_type_with_literal_and_none(self) -> None: - """str | Literal[""] | None filters Literal and marks optional.""" - result = analyze_type(str | Literal[""] | None) - - assert result.base_type == "str" - assert result.kind == TypeKind.PRIMITIVE - assert result.is_optional is True - - def test_typing_optional_sets_is_optional(self) -> None: - """Optional[str] from typing module returns TypeInfo with is_optional=True.""" - result = analyze_type(Optional[str]) # noqa: UP045 - - assert result.base_type == "str" - assert result.kind == TypeKind.PRIMITIVE - assert result.is_optional is True - assert result.is_list is False - - -class TestAnalyzeTypeUnionLiteralFiltering: - """Tests for filtering Literal arms out of unions.""" - - def test_type_with_literal_alternative(self) -> None: - """str | Literal[""] filters out the Literal and analyzes the concrete type.""" - result = analyze_type(str | Literal[""]) - - assert result.base_type == "str" - assert result.kind == TypeKind.PRIMITIVE - assert result.is_optional is False - - -class TestAnalyzeTypeList: - """Tests for list type analysis.""" - - def test_list_str_sets_is_list(self) -> None: - """list[str] returns TypeInfo with is_list=True.""" - result = analyze_type(list[str]) - - assert result.base_type == "str" - assert result.kind == TypeKind.PRIMITIVE - assert result.is_optional is False - assert result.is_list is True - - def test_nested_list_sets_depth_2(self) -> None: - """list[list[str]] records two levels of nesting.""" - result = analyze_type(list[list[str]]) - - assert result.list_depth == 2 - assert result.base_type == "str" - assert result.kind == TypeKind.PRIMITIVE - - -class TestAnalyzeTypeComposite: - """Tests for composite/nested type analysis.""" - - def test_list_optional_str(self) -> None: - """list[str | None] sets both is_list and is_optional.""" - result = analyze_type(list[str | None]) - - assert result.base_type == "str" - assert result.is_list is True - assert result.is_optional is True - - def test_optional_list_str(self) -> None: - """list[str] | None sets both is_list and is_optional.""" - result = analyze_type(list[str] | None) - - assert result.base_type == "str" - assert result.is_list is True - assert result.is_optional is True - - def test_annotated_optional_str(self) -> None: - """Annotated[str | None, ...] extracts constraints and sets is_optional.""" - result = analyze_type(Annotated[str | None, "description"]) - - assert result.base_type == "str" - assert result.is_optional is True - assert len(result.constraints) == 1 - assert result.constraints[0].source_ref is None - assert result.constraints[0].constraint == "description" - - def test_annotated_list_str(self) -> None: - """Annotated[list[str], ...] extracts constraints and sets is_list.""" - result = analyze_type(Annotated[list[str], Field(min_length=1)]) - - assert result.base_type == "str" - assert result.is_list is True - assert len(result.constraints) == 1 - assert result.constraints[0].source_ref is None - - -class TestAnalyzeTypeAnnotated: - """Tests for Annotated type analysis.""" - - def test_annotated_int_with_ge_extracts_constraint(self) -> None: - """Annotated[int, Field(ge=0)] unpacks FieldInfo to extract Ge constraint.""" - result = analyze_type(Annotated[int, Field(ge=0)]) - - assert result.base_type == "int" - assert result.kind == TypeKind.PRIMITIVE - assert len(result.constraints) == 1 - cs = result.constraints[0] - assert cs.source_ref is None + def test_filtered_leaves_concrete_type(self, missing: object) -> None: + shape = _shape(str | missing) # type: ignore[arg-type] + assert isinstance(shape, Primitive) + assert shape.base_type == "str" + assert _is_optional(str | missing) is False # type: ignore[arg-type] + + def test_with_none_sets_optional(self, missing: object) -> None: + assert _is_optional(str | missing | None) is True # type: ignore[arg-type] + + +class TestOptional: + def test_pipe_none(self) -> None: + assert _is_optional(str | None) is True + + def test_typing_optional(self) -> None: + assert _is_optional(Optional[str]) is True # noqa: UP045 + + def test_literal_arm_keeps_concrete_shape(self) -> None: + # The concrete arm is the field's shape; the literal rides along as a + # LiteralAlternatives constraint (see TestLiteralAlternatives). + shape, optional, _ = analyze_type(str | Literal[""] | None) + assert isinstance(shape, Primitive) and shape.base_type == "str" + assert optional is True + + +class TestLiteralAlternatives: + """`X | Literal[c]` keeps the concrete arm but records the literal arms.""" + + def _alternatives(self, annotation: object) -> tuple[object, ...] | None: + for cs in all_constraints(_shape(annotation)): + if isinstance(cs.constraint, LiteralAlternatives): + return cs.constraint.values + return None + + def test_scalar_literal_arm_preserved(self) -> None: + assert self._alternatives(str | Literal[""] | None) == ("",) + + def test_literal_arm_preserved_inside_list(self) -> None: + shape = _shape(list[str | Literal["x"]]) + assert isinstance(shape, ArrayOf) and isinstance(shape.element, Primitive) + assert self._alternatives(list[str | Literal["x"]]) == ("x",) + + def test_multiple_literal_values_preserved(self) -> None: + assert self._alternatives(str | Literal["a", "b"]) == ("a", "b") + + def test_pure_literal_union_has_no_alternatives(self) -> None: + # No concrete arm -> stays a LiteralScalar, no bypass constraint. + shape = _shape(Literal["a"] | None) + assert isinstance(shape, LiteralScalar) + assert self._alternatives(Literal["a"] | None) is None + + +class TestList: + def test_simple_list(self) -> None: + shape = _shape(list[str]) + assert isinstance(shape, ArrayOf) + assert isinstance(shape.element, Primitive) + assert shape.element.base_type == "str" + + def test_nested_list_records_depth(self) -> None: + shape = _shape(list[list[str]]) + assert list_depth(shape) == 2 + + def test_optional_list(self) -> None: + shape, optional, _ = analyze_type(list[str] | None) + assert isinstance(shape, ArrayOf) + assert optional is True + + def test_list_optional_element(self) -> None: + # list[str | None] is a required list whose *elements* may be None. + # Field-level optionality (list itself accepting None) is False; + # element nullability is an element-shape concern, not a field concern. + shape, optional, _ = analyze_type(list[str | None]) + assert isinstance(shape, ArrayOf) + assert optional is False + + def test_optional_list_with_optional_element(self) -> None: + # list[str | None] | None: both the field and its elements accept None. + # Field optionality is True (the outer | None), independent of the element. + shape, optional, _ = analyze_type(list[str | None] | None) + assert isinstance(shape, ArrayOf) + assert optional is True + + def test_list_inherits_element_description(self) -> None: + # A list field with no field-level description inherits its + # element's description -- losing it would leave the field + # undocumented when the only prose lives on the element. + desc = _description(list[Annotated[str, Field(description="element prose")]]) + assert desc == "element prose" + + def test_list_optional_element_desc_is_none(self) -> None: + # Element nullability alone introduces no description: list[str | None] + # has no prose on either layer, so the field description stays None. + _, _, desc = analyze_type(list[str | None]) + assert desc is None + + +class TestAnnotated: + def test_ge_collected_on_terminal(self) -> None: + shape = _shape(Annotated[int, Field(ge=0)]) + assert isinstance(shape, Primitive) + assert len(shape.constraints) == 1 + cs = shape.constraints[0] assert isinstance(cs.constraint, Ge) - assert cs.constraint.ge == 0 - - def test_annotated_without_constraints(self) -> None: - """Annotated[str, 'description'] extracts non-Field metadata.""" - result = analyze_type(Annotated[str, "just a description"]) - - assert result.base_type == "str" - assert len(result.constraints) == 1 - assert result.constraints[0].source_ref is None - assert result.constraints[0].constraint == "just a description" - - -class TestAnalyzeTypeLiteral: - """Tests for Literal type analysis.""" - - def test_literal_string_extracts_values(self) -> None: - """Literal["active"] stores the value in literal_values tuple.""" - result = analyze_type(Literal["active"]) - - assert result.kind == TypeKind.LITERAL - assert result.literal_values == ("active",) - - def test_literal_int_extracts_values(self) -> None: - """Literal[42] stores the value in literal_values tuple.""" - result = analyze_type(Literal[42]) + assert cs.source_ref is None - assert result.kind == TypeKind.LITERAL - assert result.literal_values == (42,) + def test_non_field_metadata_collected(self) -> None: + shape = _shape(Annotated[str, "just a description"]) + assert isinstance(shape, Primitive) + assert shape.constraints[0].constraint == "just a description" + + def test_list_level_minlen_lands_on_arrayof(self) -> None: + shape = _shape(Annotated[list[str], Field(min_length=1)]) + assert isinstance(shape, ArrayOf) + assert len(shape.constraints) == 1 + assert isinstance(shape.element, Primitive) + assert shape.element.constraints == () + + def test_layered_constraints_anchor_separately(self) -> None: + shape = _shape(Annotated[list[Annotated[str, MinLen(2)]], MinLen(3)]) + assert isinstance(shape, ArrayOf) + outer = shape.constraints + assert len(outer) == 1 + assert outer[0].constraint == ArrayMinLen(min_length=3) + assert isinstance(shape.element, Primitive) + inner = shape.element.constraints + assert len(inner) == 1 + assert inner[0].constraint == ScalarMinLen(min_length=2) + + +class TestAttachConstraintsOnModelTerminal: + """Constraints destined for a model/union terminal are rejected loudly.""" + + def _model_ref(self) -> FieldShape: + return ModelRef(model=RecordSpec(name="Person", description=None)) + + def _union_ref(self) -> FieldShape: + return UnionRef( + union=UnionSpec( + name="U", + description=None, + annotated_fields=[], + members=[], + discriminator_field=None, + discriminator_mapping=None, + source_annotation=object(), + common_base=BaseModel, + ) + ) - def test_multi_value_literal_stores_all_args(self) -> None: - """Literal["a", "b"] stores all args in literal_values tuple.""" - result = analyze_type(Literal["a", "b"]) + @pytest.mark.parametrize("ref_name", ["_model_ref", "_union_ref"]) + def test_constraint_on_terminal_raises(self, ref_name: str) -> None: + shape = getattr(self, ref_name)() + cs = (ConstraintSource(source_ref=None, source_name=None, constraint=Ge(0)),) + with pytest.raises(NotImplementedError): + attach_constraints(shape, cs) - assert result.kind == TypeKind.LITERAL - assert result.literal_values == ("a", "b") + @pytest.mark.parametrize("ref_name", ["_model_ref", "_union_ref"]) + def test_no_constraints_is_noop(self, ref_name: str) -> None: + shape = getattr(self, ref_name)() + assert attach_constraints(shape, ()) is shape - def test_optional_literal_extracts_values(self) -> None: - """Optional[Literal["x"]] unwraps to Literal with is_optional set.""" - result = analyze_type(Literal["x"] | None) - assert result.kind == TypeKind.LITERAL - assert result.literal_values == ("x",) - assert result.is_optional is True +class TestLiteral: + def test_single_value(self) -> None: + shape = _shape(Literal["active"]) + assert isinstance(shape, LiteralScalar) + assert shape.values == ("active",) + def test_multi_value(self) -> None: + shape = _shape(Literal["a", "b"]) + assert isinstance(shape, LiteralScalar) + assert shape.values == ("a", "b") -class TestAnalyzeTypeEnum: - """Tests for Enum type analysis.""" + def test_optional_literal(self) -> None: + shape, optional, _ = analyze_type(Literal["x"] | None) + assert isinstance(shape, LiteralScalar) + assert shape.values == ("x",) + assert optional is True - def test_enum_subclass_returns_kind_enum(self) -> None: - """Enum subclass returns TypeInfo with kind=ENUM.""" +class TestEnumAndModel: + def test_enum_emits_primitive_with_source(self) -> None: class Color(Enum): RED = "red" - GREEN = "green" - - result = analyze_type(Color) - assert result.base_type == "Color" - assert result.kind == TypeKind.ENUM - - -class TestAnalyzeTypeModel: - """Tests for BaseModel type analysis.""" - - def test_basemodel_subclass_returns_kind_model(self) -> None: - """BaseModel subclass returns TypeInfo with kind=MODEL.""" + shape = _shape(Color) + assert isinstance(shape, Primitive) + assert shape.source_type is Color + def test_model_without_resolver_falls_back_to_primitive(self) -> None: class Person(BaseModel): name: str - result = analyze_type(Person) - - assert result.base_type == "Person" - assert result.kind == TypeKind.MODEL + shape = _shape(Person) + assert isinstance(shape, Primitive) + assert shape.source_type is Person + assert shape.base_type == "Person" -class TestAnalyzeTypeNewType: - """Tests for NewType primitive analysis.""" +class TestNewType: + def test_simple_newtype(self) -> None: + shape = _shape(int32) + assert isinstance(shape, NewTypeShape) + assert shape.name == "int32" + assert isinstance(shape.inner, Primitive) + assert shape.inner.base_type == "int32" - def test_int32_returns_newtype_name(self) -> None: - """int32 NewType returns TypeInfo with base_type='int32'.""" - result = analyze_type(int32) + def test_outermost_newtype_is_outer_wrapper(self) -> None: + shape = _shape(Id) + assert isinstance(shape, NewTypeShape) + assert shape.name == "Id" - assert result.base_type == "int32" - assert result.kind == TypeKind.PRIMITIVE + def test_optional_newtype(self) -> None: + assert _is_optional(int32 | None) is True - def test_float64_returns_newtype_name(self) -> None: - """float64 NewType returns TypeInfo with base_type='float64'.""" - result = analyze_type(float64) - assert result.base_type == "float64" - assert result.kind == TypeKind.PRIMITIVE - - def test_optional_int32(self) -> None: - """int32 | None sets is_optional and preserves base_type.""" - result = analyze_type(int32 | None) - - assert result.base_type == "int32" - assert result.is_optional is True - - -class TestNewtypeName: - """Tests for outermost NewType name tracking.""" - - def test_single_layer_newtype(self) -> None: - """Single NewType like int32 sets newtype_name to its name.""" - result = analyze_type(int32) - - assert result.newtype_name == "int32" - assert result.base_type == "int32" - - def test_nested_newtype_preserves_outermost(self, id_type_info: TypeInfo) -> None: - """Nested NewType chain uses outermost name for newtype_name.""" - assert id_type_info.newtype_name == "Id" - assert id_type_info.base_type == "NoWhitespaceString" - - def test_plain_type_has_no_newtype_name(self) -> None: - """Plain types without NewType wrapping have newtype_name=None.""" - result = analyze_type(str) - - assert result.newtype_name is None - - def test_newtype_ref_set_for_newtype(self, id_type_info: TypeInfo) -> None: - """newtype_ref points to the outermost NewType callable.""" - assert id_type_info.newtype_ref is Id - - def test_newtype_ref_none_for_plain_type(self) -> None: - """Plain types have newtype_ref=None.""" - result = analyze_type(str) - - assert result.newtype_ref is None - - -class TestNewtypeWrappingList: - """Tests for NewType wrapping a list type.""" - - def test_newtype_wrapping_list(self) -> None: - """NewType wrapping a list sets is_list and preserves newtype_name.""" +class TestNewTypeWrappingList: + def test_newtype_around_list(self) -> None: TestSources = NewType("TestSources", Annotated[list[str], Field(min_length=1)]) - result = analyze_type(TestSources) + shape = _shape(TestSources) + assert isinstance(shape, NewTypeShape) and shape.name == "TestSources" + assert isinstance(shape.inner, ArrayOf) - assert result.is_list is True - assert result.newtype_name == "TestSources" + def test_list_around_scalar_newtype(self) -> None: + ScalarNT = NewType("ScalarNT", str) + shape = _shape(list[ScalarNT]) + assert isinstance(shape, ArrayOf) + assert isinstance(shape.element, NewTypeShape) - def test_scalar_newtype_is_not_list(self) -> None: - """Scalar NewType like int32 has is_list=False.""" - result = analyze_type(int32) - assert result.is_list is False +class TestConstraintProvenance: + """Constraints carry the NewType that contributed them.""" - def test_plain_list_has_no_newtype_name(self) -> None: - """Plain list[str] without NewType has newtype_name=None.""" - result = analyze_type(list[str]) + @pytest.fixture() + def id_shape(self) -> FieldShape: + return _shape(Id) - assert result.newtype_name is None - assert result.is_list is True + @pytest.fixture() + def hex_shape(self) -> FieldShape: + return _shape(HexColor) - def test_newtype_wrapping_list_of_models(self) -> None: - """list[NewType wrapping list[Model]] records depth 2, outer depth 1.""" + def test_nested_newtype_flattens_with_sources(self, id_shape: FieldShape) -> None: + sources = {cs.source_name for cs in all_constraints(id_shape)} + assert "Id" in sources + assert "NoWhitespaceString" in sources - class _Item(BaseModel): - name: str + def test_inner_newtype_constraints_preserved(self, id_shape: FieldShape) -> None: + nws = [ + cs + for cs in all_constraints(id_shape) + if cs.source_ref is NoWhitespaceString + ] + assert NoWhitespaceConstraint in {type(cs.constraint) for cs in nws} - Inner = NewType("Inner", Annotated[list[_Item], Field(min_length=1)]) - result = analyze_type(list[Inner]) + def test_direct_annotation_has_none_source(self) -> None: + shape = _shape(Annotated[str, "direct"]) + cs = all_constraints(shape) + assert len(cs) == 1 + assert cs[0].source_ref is None - assert result.list_depth == 2 - assert result.newtype_outer_list_depth == 1 - assert result.base_type == "Inner" - assert result.kind == TypeKind.MODEL - assert result.source_type is _Item + def test_single_newtype_attributed_to_itself(self, hex_shape: FieldShape) -> None: + cs = all_constraints(hex_shape) + assert cs and all(c.source_ref is HexColor for c in cs) -class TestNewtypeOuterListDepth: - """Tests for newtype_outer_list_depth tracking.""" +class TestDescription: + def test_newtype_field_description(self) -> None: + desc = _description(HexColor) + assert desc is not None and "color" in desc.lower() - def test_list_of_scalar_newtype_has_outer_depth(self) -> None: - """list[ScalarNewType] records the list layer as outside the NewType.""" - ScalarNT = NewType("ScalarNT", str) - result = analyze_type(list[ScalarNT]) - - assert result.newtype_outer_list_depth == 1 - assert result.list_depth == 1 - - def test_newtype_wrapping_list_has_zero_outer_depth(self) -> None: - """NewType wrapping list[X] records no list layers outside the NewType.""" - ListNT = NewType("ListNT", Annotated[list[str], Field(min_length=1)]) - result = analyze_type(ListNT) - - assert result.newtype_outer_list_depth == 0 - assert result.list_depth == 1 - - @pytest.mark.parametrize( - "annotation", - [ - list[str], # list without NewType - int32, # scalar NewType - str, # plain type - ], - ids=["plain_list", "scalar_newtype", "plain_type"], - ) - def test_zero_outer_depth_without_newtype_boundary( - self, annotation: object - ) -> None: - """Types without a NewType inside a list have newtype_outer_list_depth=0.""" - result = analyze_type(annotation) - - assert result.newtype_outer_list_depth == 0 - - def test_nested_list_of_scalar_newtype_has_outer_depth_2(self) -> None: - """list[list[ScalarNewType]] records two outer list layers.""" - ScalarNT = NewType("ScalarNT", str) - result = analyze_type(list[list[ScalarNT]]) + def test_plain_type_has_no_description(self) -> None: + assert _description(int) is None - assert result.newtype_outer_list_depth == 2 - assert result.list_depth == 2 + def test_annotated_field_description(self) -> None: + MyType = Annotated[str, Field(description="A test description")] + assert _description(MyType) == "A test description" + def test_outermost_description_wins(self) -> None: + desc = _description(Id) + assert desc is not None and "unique identifier" in desc.lower() -class TestConstraintProvenance: - """Tests for flattened constraints with provenance tracking.""" - - def test_nested_newtype_flattens_constraints(self, id_type_info: TypeInfo) -> None: - """Id -> NoWhitespaceString -> str flattens all constraints with sources.""" - source_names = { - cs.source_name for cs in id_type_info.constraints if cs.source_name - } - assert "Id" in source_names - assert "NoWhitespaceString" in source_names - - def test_nested_newtype_includes_inner_constraints( - self, id_type_info: TypeInfo - ) -> None: - """Inner NewType constraints are collected with provenance.""" - nws_constraints = [ - cs for cs in id_type_info.constraints if cs.source_ref is NoWhitespaceString - ] - constraint_types = {type(cs.constraint) for cs in nws_constraints} - assert NoWhitespaceConstraint in constraint_types + def test_newtype_without_field_description(self) -> None: + assert _description(SnakeCaseString) is None - def test_direct_annotation_has_none_source(self) -> None: - """Constraints from direct Annotated (no NewType) have source_ref=None.""" - result = analyze_type(Annotated[str, "direct"]) - - assert len(result.constraints) == 1 - assert result.constraints[0].source_ref is None - assert result.constraints[0].constraint == "direct" - - def test_single_newtype_constraints_attributed( - self, hex_color_type_info: TypeInfo - ) -> None: - """HexColor constraints are attributed to the HexColor callable.""" - assert all(cs.source_ref is HexColor for cs in hex_color_type_info.constraints) - assert len(hex_color_type_info.constraints) > 0 - - def test_source_ref_is_newtype_callable( - self, hex_color_type_info: TypeInfo - ) -> None: - """source_ref is the actual NewType callable, not a string.""" - cs = hex_color_type_info.constraints[0] - assert cs.source_ref is HexColor - - def test_constraint_preserves_original_object( - self, hex_color_type_info: TypeInfo - ) -> None: - """ConstraintSource.constraint holds the original constraint object.""" - hcc = next( - cs - for cs in hex_color_type_info.constraints - if type(cs.constraint).__name__ == "HexColorConstraint" - ) - assert hcc.constraint.__class__.__name__ == "HexColorConstraint" +class TestDict: + def test_simple_dict(self) -> None: + shape = _shape(dict[str, int]) + assert isinstance(shape, MapOf) + assert isinstance(shape.key, Primitive) and shape.key.base_type == "str" + assert isinstance(shape.value, Primitive) and shape.value.base_type == "int" -class TestTypeInfoDescription: - """Tests for TypeInfo.description from Field(description=...) metadata.""" + def test_optional_dict(self) -> None: + shape, optional, _ = analyze_type(dict[str, str] | None) + assert isinstance(shape, MapOf) + assert optional is True - def test_newtype_with_field_description( - self, hex_color_type_info: TypeInfo - ) -> None: - """Should extract Field description from HexColor.""" - assert hex_color_type_info.description is not None - assert "color" in hex_color_type_info.description.lower() + def test_newtype_around_dict(self) -> None: + TestMapping = NewType("TestMapping", dict[str, str]) + shape = _shape(TestMapping) + assert isinstance(shape, NewTypeShape) and shape.name == "TestMapping" + assert isinstance(shape.inner, MapOf) - def test_newtype_without_field_description(self) -> None: - """Should have None description for types without Field(description=...).""" - result = analyze_type(int) - assert result.description is None + def test_dict_with_any_value(self) -> None: + shape = _shape(dict[str, Any]) + assert isinstance(shape, MapOf) + assert isinstance(shape.value, AnyScalar) - def test_plain_annotated_with_field_description(self) -> None: - """Should extract description from Annotated with Field(description=...).""" - MyType = Annotated[str, Field(description="A test description")] - result = analyze_type(MyType) - assert result.description == "A test description" + def test_bare_dict_raises(self) -> None: + with pytest.raises(TypeError, match="Bare dict"): + analyze_type(dict) - def test_outermost_description_wins(self, id_type_info: TypeInfo) -> None: - """Outermost FieldInfo.description takes precedence in nested NewTypes.""" - assert id_type_info.description is not None - assert "unique identifier" in id_type_info.description.lower() + def test_minlen_on_map_raises(self) -> None: + with pytest.raises(NotImplementedError, match="MinLen on a Map"): + _shape(Annotated[dict[str, int], MinLen(1)]) - def test_newtype_without_field_has_none_description(self) -> None: - """NewType with constraints but no Field(description=...) has None.""" - result = analyze_type(SnakeCaseString) - assert result.description is None + def test_maxlen_on_map_raises(self) -> None: + with pytest.raises(NotImplementedError, match="MaxLen on a Map"): + _shape(Annotated[dict[str, int], MaxLen(10)]) -class TestAnalyzeTypeAny: - """Tests for typing.Any analysis.""" +class TestErrors: + def test_unsupported_annotation(self) -> None: + with pytest.raises(TypeError, match="Unsupported annotation type"): + analyze_type(42) - def test_any_returns_primitive(self) -> None: - """Any annotation returns TypeInfo with base_type='Any' and kind=PRIMITIVE.""" - result = analyze_type(Any) + def test_unresolvable_forward_ref_raises_named_error(self) -> None: + with pytest.raises( + UnresolvedForwardRefError, match="forward reference 'Missing'" + ): + analyze_type("Missing") - assert result.base_type == "Any" - assert result.kind == TypeKind.PRIMITIVE + def test_malformed_forward_ref_raises_named_error(self) -> None: + with pytest.raises(UnresolvedForwardRefError, match="not a type"): + analyze_type("not a type") - def test_dict_with_any_value(self) -> None: - """dict[str, Any] analyzes without error.""" - result = analyze_type(dict[str, Any]) + def test_dotted_forward_ref_missing_attr_raises_named_error(self) -> None: + """A dotted ref to a missing attribute fails as `UnresolvedForwardRefError`. - assert result.is_dict is True - assert result.dict_value_type is not None - assert result.dict_value_type.base_type == "Any" + `evaluate_forward_ref` raises `AttributeError` (not `NameError`) + when the head of a dotted reference resolves but the attribute is + absent; the wrapping must catch it so the failure stays named. + """ + class Outer(BaseModel): + x: int -class TestAnalyzeTypeDict: - """Tests for dict type analysis.""" + with pytest.raises( + UnresolvedForwardRefError, match="forward reference 'Outer.Missing'" + ): + analyze_type("Outer.Missing", owner=Outer) - @pytest.fixture() - def dict_str_int(self) -> TypeInfo: - return analyze_type(dict[str, int]) - - def test_dict_str_int_sets_is_dict(self, dict_str_int: TypeInfo) -> None: - """dict[str, int] returns TypeInfo with is_dict=True.""" - assert dict_str_int.is_dict is True - assert dict_str_int.is_optional is False - assert dict_str_int.is_list is False - - def test_dict_key_type_analyzed(self, dict_str_int: TypeInfo) -> None: - """dict[str, int] has dict_key_type describing the key.""" - assert dict_str_int.dict_key_type is not None - assert dict_str_int.dict_key_type.base_type == "str" - assert dict_str_int.dict_key_type.kind == TypeKind.PRIMITIVE - - def test_dict_value_type_analyzed(self, dict_str_int: TypeInfo) -> None: - """dict[str, int] has dict_value_type describing the value.""" - assert dict_str_int.dict_value_type is not None - assert dict_str_int.dict_value_type.base_type == "int" - assert dict_str_int.dict_value_type.kind == TypeKind.PRIMITIVE + def test_multi_type_union_without_resolver(self) -> None: + with pytest.raises(UnsupportedUnionError): + analyze_type(str | int) - def test_optional_dict(self) -> None: - """dict[str, str] | None sets is_dict and is_optional.""" - result = analyze_type(dict[str, str] | None) + def test_bare_list(self) -> None: + with pytest.raises(TypeError, match="Bare list without type argument"): + analyze_type(list) - assert result.is_dict is True - assert result.is_optional is True - def test_newtype_wrapping_dict(self) -> None: - """NewType wrapping dict preserves newtype_name and sets is_dict.""" - TestMapping = NewType("TestMapping", dict[str, str]) - result = analyze_type(TestMapping) +class TestForwardRefs: + def test_bare_string_element_resolved_against_owner(self) -> None: + """`list["Self"]` resolves the bare-string element via the owner namespace. - assert result.is_dict is True - assert result.newtype_name == "TestMapping" + With no `model_resolver` the resolved model terminal falls back + to a `Primitive` carrying the class as `source_type`; the point + is that the bare `str` was resolved rather than reaching the + terminal classifier unresolved. + """ - def test_bare_dict_raises_type_error(self) -> None: - """Bare dict without type arguments raises TypeError.""" - with pytest.raises(TypeError, match="Bare dict"): - analyze_type(dict) + class Node(BaseModel): + children: list["Node"] + annotation = Node.model_fields["children"].annotation + shape, _, _ = analyze_type(annotation, owner=Node) -class TestAnalyzeTypeErrors: - """Tests for error handling.""" + assert isinstance(shape, ArrayOf) + assert isinstance(shape.element, Primitive) + assert shape.element.source_type is Node - def test_unsupported_annotation_raises_type_error(self) -> None: - """Unsupported annotation type raises TypeError.""" - with pytest.raises(TypeError, match="Unsupported annotation type"): - analyze_type("not a type") + def test_nested_class_forward_ref_resolved_against_owner(self) -> None: + """A bare forward ref to a nested class resolves via the owner's namespace. - def test_multi_type_union_raises_clear_error(self) -> None: - """Multi-type unions like str | int raise UnsupportedUnionError.""" - with pytest.raises( - UnsupportedUnionError, match="Multi-type unions not supported" - ): - analyze_type(str | int) + `_resolve_forward_ref` merges `vars(owner)` into the resolution + namespace, so `"Inner"` reaches `Outer.Inner` rather than raising + `NameError`. The merge is explicit because passing any `locals` to + `evaluate_forward_ref` bypasses the library's own `vars(owner)` + fallback. + """ - def test_multi_type_union_with_none_raises_clear_error(self) -> None: - """Multi-type optional unions like str | int | None raise UnsupportedUnionError.""" - with pytest.raises( - UnsupportedUnionError, match="Multi-type unions not supported" - ): - analyze_type(str | int | None) + class Outer(BaseModel): + class Inner(BaseModel): + x: int - def test_bare_list_raises_type_error(self) -> None: - """Bare list without type argument raises TypeError.""" - with pytest.raises(TypeError, match="Bare list without type argument"): - analyze_type(list) + shape, _, _ = analyze_type("Inner", owner=Outer) + + assert isinstance(shape, Primitive) + assert shape.source_type is Outer.Inner class UnionModelA(BaseModel): @@ -598,79 +495,180 @@ class UnionModelB(BaseModel): y: str -class TestAnalyzeTypeUnion: - """Tests for discriminated union analysis.""" +class TestUnionResolver: + """Multi-arm unions of models go through the resolver callback.""" + + def test_resolver_receives_annotation_members_and_description(self) -> None: + captured: list[tuple[object, tuple[type[BaseModel], ...], str | None]] = [] + + def resolver( + annotation: object, + members: tuple[type[BaseModel], ...], + description: str | None, + ) -> Primitive: + captured.append((annotation, members, description)) + return Primitive(base_type="__captured__") - def test_all_model_union_returns_union_kind(self) -> None: - """Annotated[Union of BaseModel subclasses] returns TypeKind.UNION.""" - union_type = Annotated[UnionModelA | UnionModelB, Field(description="test")] - result = analyze_type(union_type) + union_type = Annotated[UnionModelA | UnionModelB, Field(description="x")] + shape, _, _ = analyze_type(union_type, union_resolver=resolver) - assert result.kind == TypeKind.UNION - assert result.union_members is not None - assert len(result.union_members) == 2 - assert UnionModelA in result.union_members - assert UnionModelB in result.union_members + assert isinstance(shape, Primitive) + assert shape.base_type == "__captured__" + _ann, members, description = captured[0] + expected: set[type[BaseModel]] = {UnionModelA, UnionModelB} + assert set(members) == expected + assert description == "x" + + def test_no_resolver_raises_on_multi_arm(self) -> None: + union_type = Annotated[UnionModelA | UnionModelB, Field(description="x")] + with pytest.raises(UnsupportedUnionError): + analyze_type(union_type) def test_annotated_wrapped_members_unwrapped(self) -> None: - """Union members wrapped in Annotated[X, Tag(...)] are unwrapped.""" + captured_members: list[tuple[type[BaseModel], ...]] = [] + + def resolver( + _ann: object, + members: tuple[type[BaseModel], ...], + _description: str | None, + ) -> Primitive: + captured_members.append(members) + return Primitive(base_type="x") + union_type = Annotated[ Annotated[UnionModelA, Tag("a")] | Annotated[UnionModelB, Tag("b")], Field(description="disc"), ] - result = analyze_type(union_type) - - assert result.kind == TypeKind.UNION - assert result.union_members is not None - assert len(result.union_members) == 2 - assert UnionModelA in result.union_members - assert UnionModelB in result.union_members + analyze_type(union_type, union_resolver=resolver) + expected: set[type[BaseModel]] = {UnionModelA, UnionModelB} + assert set(captured_members[0]) == expected - def test_mixed_model_nonmodel_union_still_raises(self) -> None: - """Union of model + non-model types still raises UnsupportedUnionError.""" + def test_mixed_model_nonmodel_raises(self) -> None: with pytest.raises(UnsupportedUnionError): analyze_type(UnionModelA | str) - def test_non_model_multi_union_still_raises(self) -> None: - """Multi-type union of non-models still raises UnsupportedUnionError.""" - with pytest.raises(UnsupportedUnionError): - analyze_type(str | int) - - def test_union_base_type_is_first_member_name(self) -> None: - """UNION TypeInfo base_type is the first member's class name.""" - result = analyze_type( - Annotated[UnionModelA | UnionModelB, Field(description="test")] - ) - assert result.base_type == "UnionModelA" - - def test_optional_union_sets_is_optional(self) -> None: - """Union with None among model members sets is_optional.""" - result = analyze_type( - Annotated[UnionModelA | UnionModelB, Field(description="test")] | None - ) - assert result.kind == TypeKind.UNION - assert result.is_optional is True - class TestSingleLiteralValue: - """Tests for single_literal_value convenience accessor.""" - - def test_single_value_literal(self) -> None: - """Literal["x"] returns the literal value.""" + def test_single_string(self) -> None: assert single_literal_value(Literal["x"]) == "x" - def test_single_int_literal(self) -> None: - """Literal[42] returns the integer value.""" + def test_single_int(self) -> None: assert single_literal_value(Literal[42]) == 42 - def test_multi_value_literal_returns_none(self) -> None: - """Multi-value Literal returns None (no single default).""" + def test_multi_value_returns_none(self) -> None: assert single_literal_value(Literal["a", "b"]) is None def test_non_literal_returns_none(self) -> None: - """Non-Literal types return None.""" assert single_literal_value(str) is None - def test_unsupported_type_returns_none(self) -> None: - """Types that raise during analysis return None.""" + def test_unsupported_returns_none(self) -> None: assert single_literal_value("not a type") is None + + +class TestUnwrapList: + def test_plain_list(self) -> None: + assert unwrap_list(list[int]) is int + + def test_nested_list(self) -> None: + assert unwrap_list(list[list[str]]) is str + + def test_non_list_passthrough(self) -> None: + assert unwrap_list(int) is int + + def test_optional_list(self) -> None: + assert unwrap_list(list[int] | None) is int + + def test_optional_list_preserves_annotated(self) -> None: + assert unwrap_list(list[VehicleSelector] | None) is VehicleSelector + + +class TestNestedArrayCharacterization: + """Pin analyze_type behavior on consecutive-list and NewType-chain shapes. + + The schema has no genuine `list[list[X]]` field, so these are the only + coverage of the path the recursive _unwrap rewrite must preserve. + """ + + def test_list_of_list_nests_two_arrayofs(self) -> None: + shape = _shape(list[list[str]]) + assert isinstance(shape, ArrayOf) + assert isinstance(shape.element, ArrayOf) + assert isinstance(shape.element.element, Primitive) + assert shape.element.element.base_type == "str" + + def test_list_of_list_constraints_anchor_to_their_layer(self) -> None: + # Each MinLen lands on the ArrayOf layer it annotates, not flattened. + # Outer Annotated[..., Field(min_length=3)] targets the outer list. + # Inner Annotated[list[str], Field(min_length=2)] targets the inner list. + shape = _shape( + Annotated[ + list[Annotated[list[str], Field(min_length=2)]], Field(min_length=3) + ] + ) + assert isinstance(shape, ArrayOf) + inner = shape.element + assert isinstance(inner, ArrayOf) + outer_min_lens = [ + cs.constraint.min_length + for cs in shape.constraints + if isinstance(cs.constraint, ArrayMinLen) + ] + inner_min_lens = [ + cs.constraint.min_length + for cs in inner.constraints + if isinstance(cs.constraint, ArrayMinLen) + ] + assert outer_min_lens == [3] + assert inner_min_lens == [2] + + def test_nested_newtype_chain_flattens_to_one_wrapper(self) -> None: + # Id = NewType("Id", Annotated[NoWhitespaceString, Field(min_length=1)]) + shape = _shape(Id) + assert isinstance(shape, NewTypeShape) + assert shape.name == "Id" + # exactly one NewTypeShape -- the inner NoWhitespaceString does not nest + assert not isinstance(shape.inner, NewTypeShape) + assert isinstance(shape.inner, Primitive) + assert shape.inner.base_type == "NoWhitespaceString" + + def test_nested_newtype_constraint_order_outer_first(self) -> None: + shape = _shape(Id) + names = [cs.source_name for cs in all_constraints(shape)] + # Id's own constraint precedes NoWhitespaceString's + assert names == ["Id", "NoWhitespaceString"] + + def test_newtype_nested_as_list_element_flattens_under_outer_newtype(self) -> None: + # A NewType chain collapses to one NewTypeShape (the outermost) even + # when an inner NewType is nested across a list boundary -- the inner + # name survives only as the terminal `base_type`. + InnerElem = NewType("InnerElem", str) + OuterList = NewType("OuterList", list[InnerElem]) + shape = _shape(OuterList) + assert isinstance(shape, NewTypeShape) + assert shape.name == "OuterList" + assert isinstance(shape.inner, ArrayOf) + # the InnerElem NewType does NOT produce its own NewTypeShape + assert isinstance(shape.inner.element, Primitive) + assert shape.inner.element.base_type == "InnerElem" + + def test_sole_list_element_newtype_keeps_its_wrapper(self) -> None: + # With no outer NewType, a list-element NewType IS the outermost -- + # it keeps its NewTypeShape (guards against over-erasing). + ElemOnly = NewType("ElemOnly", str) + shape = _shape(list[ElemOnly]) + assert isinstance(shape, ArrayOf) + assert isinstance(shape.element, NewTypeShape) + assert shape.element.name == "ElemOnly" + + def test_newtype_inside_dict_value_is_an_independent_spine(self) -> None: + # `dict` key/value are independent spines: a NewType in the value + # keeps its wrapper even under an outer NewType, because erasure + # stops at MapOf. + DictValue = NewType("DictValue", str) + DictWrap = NewType("DictWrap", dict[str, DictValue]) + shape = _shape(DictWrap) + assert isinstance(shape, NewTypeShape) + assert shape.name == "DictWrap" + assert isinstance(shape.inner, MapOf) + assert isinstance(shape.inner.value, NewTypeShape) + assert shape.inner.value.name == "DictValue" diff --git a/packages/overture-schema-codegen/tests/test_type_collection.py b/packages/overture-schema-codegen/tests/test_type_collection.py index 154b39e2c..66cbbcbd5 100644 --- a/packages/overture-schema-codegen/tests/test_type_collection.py +++ b/packages/overture-schema-codegen/tests/test_type_collection.py @@ -8,22 +8,21 @@ TestSegmentWithSubModel, has_name, lookup_by_name, -) -from overture.schema.codegen.extraction.model_extraction import ( - expand_model_tree, - extract_model, + spec_for_model, ) from overture.schema.codegen.extraction.specs import ( EnumSpec, - ModelSpec, NewTypeSpec, PydanticTypeSpec, + RecordSpec, SupplementarySpec, TypeIdentity, ) from overture.schema.codegen.layout.type_collection import ( collect_all_supplementary_types, ) +from overture.schema.system.primitive import uint8 +from overture.schema.system.string import HexColor from pydantic import BaseModel @@ -37,9 +36,7 @@ def _make_feature_with_sub_model(sub_model: type) -> type[BaseModel]: def _expanded_supplementary(model_class: type) -> dict[TypeIdentity, SupplementarySpec]: - spec = extract_model(model_class) - expand_model_tree(spec) - return collect_all_supplementary_types([spec]) + return collect_all_supplementary_types([spec_for_model(model_class)]) class TestCollectAllSupplementarySpecs: @@ -61,7 +58,7 @@ def test_returns_model_specs_from_expanded_tree(self) -> None: result = _expanded_supplementary(FeatureWithAddress) assert has_name(result, "Address") - assert isinstance(lookup_by_name(result, "Address"), ModelSpec) + assert isinstance(lookup_by_name(result, "Address"), RecordSpec) def test_collects_transitive_types(self) -> None: """Types referenced by sub-models are also collected.""" @@ -77,11 +74,8 @@ def test_same_name_different_types_both_collected(self) -> None: ModelA = type("Address", (BaseModel,), {"__annotations__": {"x": str}}) ModelB = type("Address", (BaseModel,), {"__annotations__": {"y": int}}) - outer_a = extract_model(_make_feature_with_sub_model(ModelA)) - expand_model_tree(outer_a) - - outer_b = extract_model(_make_feature_with_sub_model(ModelB)) - expand_model_tree(outer_b) + outer_a = spec_for_model(_make_feature_with_sub_model(ModelA)) + outer_b = spec_for_model(_make_feature_with_sub_model(ModelB)) result = collect_all_supplementary_types([outer_a, outer_b]) @@ -103,7 +97,7 @@ class FeatureWithUnionSubModel(BaseModel): result = _expanded_supplementary(FeatureWithUnionSubModel) assert has_name(result, "ContactInfo") - assert isinstance(lookup_by_name(result, "ContactInfo"), ModelSpec) + assert isinstance(lookup_by_name(result, "ContactInfo"), RecordSpec) class TestCollectPydanticTypes: @@ -126,3 +120,21 @@ def test_does_not_collect_builtin_primitives(self) -> None: result = _expanded_supplementary(FeatureWithUrl) assert not has_name(result, "str") assert not has_name(result, "int") + + +class TestSemanticNewtypeGuard: + """Only semantic NewTypes become standalone supplementary specs.""" + + def test_registered_primitive_newtype_not_collected(self) -> None: + """A non-semantic NewType (uint8) belongs on the aggregate + primitives page, not as a standalone NewTypeSpec whose path + collides with it.""" + feature = type( + "FeatureWithPrimitives", + (BaseModel,), + {"__annotations__": {"count": uint8, "color": HexColor}}, + ) + result = _expanded_supplementary(feature) + + assert not has_name(result, "uint8") + assert has_name(result, "HexColor") diff --git a/packages/overture-schema-codegen/tests/test_type_placement.py b/packages/overture-schema-codegen/tests/test_type_placement.py index 63e26457c..5ae07dbbd 100644 --- a/packages/overture-schema-codegen/tests/test_type_placement.py +++ b/packages/overture-schema-codegen/tests/test_type_placement.py @@ -11,12 +11,11 @@ lookup_by_name, make_union_spec, ) -from overture.schema.codegen.extraction.model_extraction import expand_model_tree from overture.schema.codegen.extraction.specs import ( AnnotatedField, - FeatureSpec, FieldSpec, ModelSpec, + RecordSpec, SupplementarySpec, TypeIdentity, ) @@ -42,15 +41,12 @@ def _build_registry( - feature_specs: list[ModelSpec], + model_specs: list[RecordSpec], ) -> tuple[dict[TypeIdentity, PurePosixPath], dict[TypeIdentity, SupplementarySpec]]: """Build placement registry with standard aggregate names.""" - cache: dict[type, ModelSpec] = {} - for spec in feature_specs: - expand_model_tree(spec, cache) - all_specs = collect_all_supplementary_types(feature_specs) + all_specs = collect_all_supplementary_types(model_specs) registry = build_placement_registry( - feature_specs, all_specs, _NUMERIC_NAMES, _GEOMETRY_NAMES, _SCHEMA_ROOT + model_specs, all_specs, _NUMERIC_NAMES, _GEOMETRY_NAMES, _SCHEMA_ROOT ) return registry, all_specs @@ -149,7 +145,7 @@ class TestPlacementWithUnionSpec: """Tests for placement registry with UnionSpec.""" def test_union_spec_gets_placement(self) -> None: - """UnionSpec is placed alongside ModelSpec in the registry.""" + """UnionSpec is placed alongside RecordSpec in the registry.""" class Base(BaseModel): name: str @@ -162,7 +158,7 @@ class A(Base): AnnotatedField( field_spec=FieldSpec( name="name", - type_info=STR_TYPE, + shape=STR_TYPE, description=None, is_required=True, ), @@ -174,10 +170,10 @@ class A(Base): entry_point="test.package:TestUnion", ) - feature_specs: list[FeatureSpec] = [union_spec] - all_specs = collect_all_supplementary_types(feature_specs) + model_specs: list[ModelSpec] = [union_spec] + all_specs = collect_all_supplementary_types(model_specs) registry = build_placement_registry( - feature_specs, all_specs, [], [], "test.package" + model_specs, all_specs, [], [], "test.package" ) assert any(tid.name == "TestUnion" for tid in registry) @@ -206,7 +202,7 @@ class TestPydanticTypePlacement: def test_pydantic_type_placed_under_module_dir(self) -> None: registry = build_placement_registry( - feature_specs=[], + model_specs=[], all_specs={HTTP_URL_SPEC.identity: HTTP_URL_SPEC}, numeric_names=[], geometry_names=[], @@ -222,7 +218,7 @@ def test_multiple_pydantic_types_same_module(self) -> None: EMAIL_STR_SPEC.identity: EMAIL_STR_SPEC, } registry = build_placement_registry( - feature_specs=[], + model_specs=[], all_specs=specs, numeric_names=[], geometry_names=[], diff --git a/packages/overture-schema-codegen/tests/test_type_registry.py b/packages/overture-schema-codegen/tests/test_type_registry.py index b9d02d2ac..3ca2e01c7 100644 --- a/packages/overture-schema-codegen/tests/test_type_registry.py +++ b/packages/overture-schema-codegen/tests/test_type_registry.py @@ -1,43 +1,46 @@ """Tests for type registry.""" -import pytest -from overture.schema.codegen.extraction.type_analyzer import TypeInfo, TypeKind +from overture.schema.codegen.extraction.field import ( + ArrayOf, + NewTypeShape, + Primitive, +) from overture.schema.codegen.extraction.type_registry import ( PRIMITIVE_TYPES, TypeMapping, get_type_mapping, + primitive_spark_category, resolve_type_name, ) class TestTypeMapping: - """Tests for TypeMapping dataclass.""" - - def test_typemapping_accepts_markdown(self) -> None: - """TypeMapping should construct with markdown field.""" - mapping = TypeMapping(markdown="int32") - - assert mapping.markdown == "int32" - - def test_for_target_returns_markdown(self) -> None: - """for_target should return markdown representation for markdown target.""" - mapping = TypeMapping(markdown="int32") - - assert mapping.for_target("markdown") == "int32" - - def test_for_target_rejects_unknown_target(self) -> None: - """for_target should raise ValueError for unknown targets.""" - mapping = TypeMapping(markdown="int32") - - with pytest.raises(ValueError, match="Unknown target 'scala'"): - mapping.for_target("scala") + def test_markdown_field(self) -> None: + assert TypeMapping(markdown="int32").markdown == "int32" + + def test_spark_type_mapping(self) -> None: + cases = [ + ("str", "StringType()"), + ("int32", "IntegerType()"), + ("int64", "LongType()"), + ("float64", "DoubleType()"), + ("bool", "BooleanType()"), + ("Geometry", "BinaryType()"), + ("float32", "FloatType()"), + ] + for type_name, expected in cases: + mapping = get_type_mapping(type_name) + assert mapping is not None, f"No mapping for {type_name!r}" + assert mapping.spark == expected + + def test_bbox_has_no_spark_mapping(self) -> None: + mapping = get_type_mapping("BBox") + assert mapping is not None + assert mapping.spark is None class TestPrimitiveTypes: - """Tests for PRIMITIVE_TYPES registry.""" - def test_registry_contains_expected_types(self) -> None: - """Registry should contain all expected primitive types.""" expected_types = { "int8", "int16", @@ -55,89 +58,79 @@ def test_registry_contains_expected_types(self) -> None: "Geometry", "BBox", } - assert set(PRIMITIVE_TYPES.keys()) == expected_types def test_bbox_mapping(self) -> None: - """BBox should map to bbox.""" bbox = PRIMITIVE_TYPES["BBox"] - assert bbox.markdown == "bbox" + assert bbox.spark is None class TestGetTypeMapping: - """Tests for get_type_mapping function.""" - def test_returns_mapping_for_known_type(self) -> None: - """Should return TypeMapping for known primitive type.""" - result = get_type_mapping("int32") - - assert result is not None - assert result.markdown == "int32" + assert get_type_mapping("int32").markdown == "int32" # type: ignore[union-attr] def test_returns_none_for_unknown_type(self) -> None: - """Should return None for unknown type names.""" - result = get_type_mapping("unknown_type") - - assert result is None + assert get_type_mapping("unknown_type") is None def test_returns_mapping_for_builtin_int(self) -> None: - """Should map Python int to int64.""" - result = get_type_mapping("int") + assert get_type_mapping("int").markdown == "int64" # type: ignore[union-attr] - assert result is not None - assert result.markdown == "int64" - def test_returns_mapping_for_builtin_float(self) -> None: - """Should map Python float to float64.""" - result = get_type_mapping("float") +class TestResolveTypeName: + def test_unregistered_newtype_falls_back_to_source_type(self) -> None: + cls = type("SourceItem", (), {}) + shape = NewTypeShape( + name="Sources", + ref=object(), + inner=Primitive(base_type="Sources", source_type=cls), + ) + assert resolve_type_name(shape) == "SourceItem" - assert result is not None - assert result.markdown == "float64" + def test_registered_newtype_resolves_via_registry(self) -> None: + shape = NewTypeShape( + name="int32", + ref=object(), + inner=Primitive(base_type="int32", source_type=int), + ) + assert resolve_type_name(shape) == "int32" + def test_plain_scalar(self) -> None: + assert ( + resolve_type_name(Primitive(base_type="str", source_type=str)) == "string" + ) -class TestResolveTypeNameNewTypeFallback: - """Tests for resolve_type_name with unregistered NewTypes.""" + def test_array_of_scalar_resolves_terminal(self) -> None: + shape = ArrayOf(element=Primitive(base_type="str", source_type=str)) + assert resolve_type_name(shape) == "string" - def test_unregistered_newtype_falls_back_to_source_type(self) -> None: - """Unregistered NewType resolves to source_type name.""" - ti = TypeInfo( - base_type="Sources", - kind=TypeKind.MODEL, - newtype_name="Sources", - source_type=type("SourceItem", (), {}), - ) - result = resolve_type_name(ti, "markdown") - assert result == "SourceItem" +class TestPrimitiveSparkCategory: + def test_int_types_are_int(self) -> None: + for bt in ( + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "int", + ): + assert primitive_spark_category(bt) == "int", bt - def test_registered_newtype_unaffected(self) -> None: - """Registered NewType (int32) still resolves through the registry.""" - ti = TypeInfo( - base_type="int32", - kind=TypeKind.PRIMITIVE, - newtype_name="int32", - source_type=int, - ) - result = resolve_type_name(ti, "markdown") + def test_float_types_are_float(self) -> None: + for bt in ("float32", "float64", "float"): + assert primitive_spark_category(bt) == "float", bt - assert result == "int32" + def test_bool_is_bool(self) -> None: + assert primitive_spark_category("bool") == "bool" + def test_string_type_is_string(self) -> None: + assert primitive_spark_category("str") == "string" -class TestResolveTypeName: - """Tests for resolve_type_name with list/optional flags.""" - - def _make_type_info(self, **kwargs: object) -> TypeInfo: - defaults = {"base_type": "str", "kind": TypeKind.PRIMITIVE} - defaults.update(kwargs) - return TypeInfo(**defaults) # type: ignore[arg-type] - - def test_ignores_list_depth(self) -> None: - """resolve_type_name returns the base type regardless of list_depth.""" - ti = self._make_type_info(list_depth=1) - assert resolve_type_name(ti, "markdown") == "string" - - def test_ignores_is_optional(self) -> None: - """resolve_type_name returns the base type regardless of is_optional.""" - ti = self._make_type_info(is_optional=True) - assert resolve_type_name(ti, "markdown") == "string" + def test_unknown_type_falls_back_to_other(self) -> None: + assert primitive_spark_category("UnknownNewType") == "other" + + def test_geometry_is_other(self) -> None: + assert primitive_spark_category("Geometry") == "other" diff --git a/packages/overture-schema-codegen/tests/test_union_extraction.py b/packages/overture-schema-codegen/tests/test_union_extraction.py index a8b685c48..73f599332 100644 --- a/packages/overture-schema-codegen/tests/test_union_extraction.py +++ b/packages/overture-schema-codegen/tests/test_union_extraction.py @@ -1,15 +1,42 @@ """Tests for union extraction.""" +import re +from typing import Any + import pytest +from annotated_types import MinLen from codegen_test_support import ( RailSegment, RoadSegment, SegmentBase, + TestEnumDiscriminatorUnion, TestSegment, + TestSegmentDivergingConstraints, + TestSegmentEqualConstraints, WaterSegment, ) +from overture.schema.codegen.extraction.field import ( + ArrayOf, + ConstraintSource, + Primitive, +) from overture.schema.codegen.extraction.specs import FieldSpec, UnionSpec -from overture.schema.codegen.extraction.union_extraction import extract_union +from overture.schema.codegen.extraction.union_extraction import ( + _constraints_fingerprint, + extract_union, +) +from overture.schema.common.scoping.vehicle import VehicleSelector +from overture.schema.system.field_constraint import ( + FieldConstraint, + UniqueItemsConstraint, +) +from overture.schema.system.field_constraint.string import ( + CountryCodeAlpha2Constraint, + JsonPointerConstraint, + PatternConstraint, +) +from pydantic import Field, GetCoreSchemaHandler +from pydantic_core import core_schema class TestExtractUnion: @@ -51,19 +78,19 @@ def test_shared_fields_first(self, segment_spec: UnionSpec) -> None: def test_variant_specific_fields_have_sources( self, segment_spec: UnionSpec ) -> None: - """Variant-only fields carry their source class names.""" + """Variant-only fields carry their source classes.""" speed = next( af for af in segment_spec.annotated_fields if af.field_spec.name == "speed_limit" ) - assert speed.variant_sources == ("RoadSegment",) + assert speed.variant_sources == (RoadSegment,) gauge = next( af for af in segment_spec.annotated_fields if af.field_spec.name == "rail_gauge" ) - assert gauge.variant_sources == ("RailSegment",) + assert gauge.variant_sources == (RailSegment,) def test_heterogeneous_same_name_produces_separate_rows( self, segment_spec: UnionSpec @@ -74,8 +101,8 @@ def test_heterogeneous_same_name_produces_separate_rows( ] assert len(class_fields) == 2 sources = {af.variant_sources for af in class_fields} - assert ("RoadSegment",) in sources - assert ("RailSegment",) in sources + assert (RoadSegment,) in sources + assert (RailSegment,) in sources def test_members_lists_all_member_classes(self, segment_spec: UnionSpec) -> None: """UnionSpec.members contains all union member classes.""" @@ -89,3 +116,160 @@ def test_fields_property_returns_plain_list(self, segment_spec: UnionSpec) -> No """spec.fields returns list[FieldSpec] without provenance.""" for f in segment_spec.fields: assert isinstance(f, FieldSpec) + + +class TestExtractDiscriminatorWithEnumLiterals: + """Discriminator mapping uses runtime string values for enum literals.""" + + @pytest.fixture + def spec(self) -> UnionSpec: + return extract_union("TestEnumDiscriminatorUnion", TestEnumDiscriminatorUnion) + + def test_discriminator_mapping_uses_enum_values(self, spec: UnionSpec) -> None: + """Mapping keys must be the Parquet-serialized string values, not enum repr.""" + assert spec.discriminator_mapping is not None + assert set(spec.discriminator_mapping.keys()) == {"car", "bike"} + + +class TestDivergingConstraints: + """Same-named fields with matching shape but diverging constraints fail loudly.""" + + def test_diverging_constraints_raise(self) -> None: + """A field shared by structure but not by constraints raises ValueError. + + `ShortNamesSegment` and `LongNamesSegment` both declare `aliases` + as `list[str] | None`, so the structural fingerprint collapses + them — but the `min_length` constraints differ. Dedup would + silently keep one member's `FieldSpec`, so extraction raises + instead. + """ + with pytest.raises(ValueError, match="diverging constraints"): + extract_union( + "TestSegmentDivergingConstraints", TestSegmentDivergingConstraints + ) + + +class TestUnionNameDerivation: + """Union name fallback when the caller passes a member class name.""" + + def test_name_derived_from_common_base(self) -> None: + """When name matches a member class, derive from common base minus 'Base' suffix.""" + spec = extract_union("VehicleAxleCountSelector", VehicleSelector) + assert spec.name == "VehicleSelector" + + +def _make_array_field(constraint: object) -> FieldSpec: + """Build a FieldSpec with one array-level constraint for fingerprint tests.""" + cs = ConstraintSource(source_ref=None, source_name=None, constraint=constraint) + return FieldSpec( + name="items", shape=ArrayOf(element=Primitive("str"), constraints=(cs,)) + ) + + +def _make_scalar_field(constraint: object) -> FieldSpec: + """Build a FieldSpec with one scalar-level constraint for fingerprint tests.""" + cs = ConstraintSource(source_ref=None, source_name=None, constraint=constraint) + return FieldSpec(name="tag", shape=Primitive("str", constraints=(cs,))) + + +class _ListAttrConstraint(FieldConstraint): + """Test-only constraint with a list-valued attribute (fingerprint hashability guard).""" + + def __init__(self, items: list[str]) -> None: + self.items = list(items) + + def __get_pydantic_core_schema__( + self, source: type[Any], handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return handler(source) + + +class TestConstraintsFingerprint: + """_constraints_fingerprint produces value-stable keys across distinct instances.""" + + def test_marker_constraint_equal_instances_same_fingerprint(self) -> None: + """Two distinct UniqueItemsConstraint() instances fingerprint equal.""" + fs1 = _make_array_field(UniqueItemsConstraint()) + fs2 = _make_array_field(UniqueItemsConstraint()) + assert _constraints_fingerprint(fs1) == _constraints_fingerprint(fs2) + + def test_parametric_constraint_equal_instances_same_fingerprint(self) -> None: + """Two distinct CountryCodeAlpha2Constraint() instances fingerprint equal.""" + fs1 = _make_scalar_field(CountryCodeAlpha2Constraint()) + fs2 = _make_scalar_field(CountryCodeAlpha2Constraint()) + assert _constraints_fingerprint(fs1) == _constraints_fingerprint(fs2) + + def test_different_attribute_values_unequal_fingerprint(self) -> None: + """Constraints differing in attribute value produce different fingerprints.""" + fs1 = _make_array_field(MinLen(1)) + fs2 = _make_array_field(MinLen(5)) + assert _constraints_fingerprint(fs1) != _constraints_fingerprint(fs2) + + def test_equal_constraints_do_not_raise(self) -> None: + """Union extraction does not raise when members share a field with equal constraints.""" + extract_union("TestSegmentEqualConstraints", TestSegmentEqualConstraints) + + def test_different_zero_attr_constraint_classes_unequal(self) -> None: + """Two different marker constraint classes with no attributes fingerprint unequal. + + Both `UniqueItemsConstraint()` and `JsonPointerConstraint()` have no + instance attributes, so their keys differ only by qualified class name. + """ + fs1 = _make_array_field(UniqueItemsConstraint()) + fs2 = _make_array_field(JsonPointerConstraint()) + assert _constraints_fingerprint(fs1) != _constraints_fingerprint(fs2) + + def test_pattern_constraint_flags_distinguish_fingerprint(self) -> None: + """PatternConstraints with the same source but different flags diverge. + + Value equality on `PatternConstraint` normalizes the compiled + `re.Pattern` to `(pattern, flags)`, so a flag difference produces + distinct fingerprints rather than collapsing. + """ + fs1 = _make_scalar_field(PatternConstraint(r"^[a-z]+$", "err")) + fs2 = _make_scalar_field(PatternConstraint(r"^[a-z]+$", "err", re.IGNORECASE)) + assert _constraints_fingerprint(fs1) != _constraints_fingerprint(fs2) + + def test_distinct_value_eq_constraints_diverge(self) -> None: + """Same type, different attributes diverge -- the case dedup guards. + + Per-variant divergence on a structurally identical field is the + condition `_constraints_fingerprint` exists to catch. Two + `PatternConstraint`s differing only in source pattern must not collapse. + """ + fs1 = _make_scalar_field(PatternConstraint(r"^[a-z]+$", "err")) + fs2 = _make_scalar_field(PatternConstraint(r"^[0-9]+$", "err")) + assert _constraints_fingerprint(fs1) != _constraints_fingerprint(fs2) + + def test_container_valued_constraint_routes_through_fingerprint(self) -> None: + """A container-valued constraint builds a frozenset without raising. + + Guards the original failure mode: an unhashable constraint key crashed + `frozenset` construction in `_constraints_fingerprint`. Value `__hash__` + on the constraint normalizes the list attribute, so equal instances + both hash and collapse. + """ + fs1 = _make_array_field(_ListAttrConstraint(["a", "b"])) + fs2 = _make_array_field(_ListAttrConstraint(["a", "b"])) + assert _constraints_fingerprint(fs1) == _constraints_fingerprint(fs2) + + def test_foreign_identity_eq_metadata_equal_instances_collapse(self) -> None: + """Raw pydantic `Field(...)` metadata compares by identity but collapses. + + Pydantic's internal metadata is the lone constraint type that falls + back to identity equality, so two equal-valued instances would + fingerprint as divergent. `_fingerprint_key` keys it on its + value-stable `repr` so equal metadata still collapses. + """ + raw1 = Field(pattern=r"^[a-z]+$").metadata[0] + raw2 = Field(pattern=r"^[a-z]+$").metadata[0] + assert raw1 != raw2 + fs1 = _make_scalar_field(raw1) + fs2 = _make_scalar_field(raw2) + assert _constraints_fingerprint(fs1) == _constraints_fingerprint(fs2) + + def test_foreign_identity_eq_metadata_distinct_values_diverge(self) -> None: + """Different raw `Field(...)` patterns produce divergent fingerprints.""" + fs1 = _make_scalar_field(Field(pattern=r"^[a-z]+$").metadata[0]) + fs2 = _make_scalar_field(Field(pattern=r"^[0-9]+$").metadata[0]) + assert _constraints_fingerprint(fs1) != _constraints_fingerprint(fs2) diff --git a/packages/overture-schema-common/src/overture/schema/common/scoping/vehicle.py b/packages/overture-schema-common/src/overture/schema/common/scoping/vehicle.py index 801d35aa3..287b25c55 100644 --- a/packages/overture-schema-common/src/overture/schema/common/scoping/vehicle.py +++ b/packages/overture-schema-common/src/overture/schema/common/scoping/vehicle.py @@ -38,24 +38,30 @@ class VehicleRelation(str, Enum): @no_extra_fields -class VehicleAxleCountSelector(BaseModel): +class VehicleSelectorBase(BaseModel): """ - Selects vehicles based on the number of axles they have. + Common fields shared by all vehicle selector subtypes. + + See also: `VehicleSelector`. """ - dimension: Literal[VehicleDimension.AXLE_COUNT] + dimension: VehicleDimension comparison: VehicleRelation + + +@no_extra_fields +class VehicleAxleCountSelector(VehicleSelectorBase): + """Selects vehicles based on the number of axles they have.""" + + dimension: Literal[VehicleDimension.AXLE_COUNT] value: uint8 = Field(description="Number of axles on the vehicle") @no_extra_fields -class VehicleHeightSelector(BaseModel): - """ - Selects vehicles based on their height. - """ +class VehicleHeightSelector(VehicleSelectorBase): + """Selects vehicles based on their height.""" dimension: Literal[VehicleDimension.HEIGHT] - comparison: VehicleRelation value: Annotated[ float64, Field( @@ -66,13 +72,10 @@ class VehicleHeightSelector(BaseModel): @no_extra_fields -class VehicleLengthSelector(BaseModel): - """ - Selects vehicles based on their length. - """ +class VehicleLengthSelector(VehicleSelectorBase): + """Selects vehicles based on their length.""" dimension: Literal[VehicleDimension.LENGTH] - comparison: VehicleRelation value: Annotated[ float64, Field( @@ -83,13 +86,10 @@ class VehicleLengthSelector(BaseModel): @no_extra_fields -class VehicleWeightSelector(BaseModel): - """ - Selects vehicles based on their weight. - """ +class VehicleWeightSelector(VehicleSelectorBase): + """Selects vehicles based on their weight.""" dimension: Literal[VehicleDimension.WEIGHT] - comparison: VehicleRelation value: Annotated[ float64, Field( @@ -100,13 +100,10 @@ class VehicleWeightSelector(BaseModel): @no_extra_fields -class VehicleWidthSelector(BaseModel): - """ - Selects vehicles based on their width. - """ +class VehicleWidthSelector(VehicleSelectorBase): + """Selects vehicles based on their width.""" dimension: Literal[VehicleDimension.WIDTH] - comparison: VehicleRelation value: Annotated[ float64, Field( @@ -123,7 +120,8 @@ class VehicleWidthSelector(BaseModel): | VehicleWeightSelector | VehicleWidthSelector, Field( - description="Selects vehicles that a scope applies to based on criteria such as height, weight, or axle count." + discriminator="dimension", + description="Selects vehicles that a scope applies to based on criteria such as height, weight, or axle count.", ), ] """ diff --git a/packages/overture-schema-divisions-theme/pyproject.toml b/packages/overture-schema-divisions-theme/pyproject.toml index 0314d8d1b..ccec15fd7 100644 --- a/packages/overture-schema-divisions-theme/pyproject.toml +++ b/packages/overture-schema-divisions-theme/pyproject.toml @@ -1,6 +1,6 @@ [project] maintainers = [ - {name = "Overture Maps Schema Working Group"}, + { name = "Overture Maps Schema Working Group" }, ] dependencies = [ "overture-schema-common", @@ -42,111 +42,126 @@ division_boundary = "overture.schema.divisions:DivisionBoundary" overture_baselines = "overture.schema.system.testing.plugin" [[examples.Division]] -id = "350e85f6-68ba-4114-9906-c2844815988b" -geometry = "POINT (-175.2551522 -21.1353686)" -country = "TO" -version = 1 -subtype = "locality" -class = "village" -region = "TO-04" +admin_level = 2 +capital_division_ids = ["958c67d3-868a-482d-9dd8-1c65b0c1a9e8"] +country = "US" +geometry = "POINT (-159.4945109 22.0557204)" hierarchies = [ - [ - {division_id = "fef8748b-0c91-46ad-9f2d-976d8d2de3e9", subtype = "country", name = "Tonga"}, - {division_id = "4d67561a-2292-41bd-8996-7853d276a42c", subtype = "region", name = "Tongatapu"}, - {division_id = "8730f0cc-d436-4f11-a7d3-49085813ef44", subtype = "county", name = "Vahe Kolomotu'a"}, - {division_id = "350e85f6-68ba-4114-9906-c2844815988b", subtype = "locality", name = "Sia'atoutai"}, - ], + [ + { division_id = "f39eb4af-5206-481b-b19e-bd784ded3f05", subtype = "country", name = "United States" }, + { division_id = "a2a08395-e968-4be5-bdd7-7f63db8c2165", subtype = "region", name = "Hawaii" }, + { division_id = "c9b8adc9-4639-4392-8efe-8401eeb19929", subtype = "county", name = "Kauaʻi County" }, + ], ] -parent_division_id = "8730f0cc-d436-4f11-a7d3-49085813ef44" -population = 534 +id = "c9b8adc9-4639-4392-8efe-8401eeb19929" +parent_division_id = "a2a08395-e968-4be5-bdd7-7f63db8c2165" +population = 71735 +region = "US-HI" +subtype = "county" theme = "divisions" type = "division" +version = 5 +wikidata = "Q111517" [examples.Division.bbox] -xmin = -175.25515747070312 -xmax = -175.255126953125 -ymin = -21.1353702545166 -ymax = -21.13536834716797 +xmax = -159.4945068359375 +xmin = -159.49453735351562 +ymax = 22.055721282958984 +ymin = 22.05571746826172 [[examples.Division.sources]] -property = "" dataset = "OpenStreetMap" -record_id = "n3173231082@4" -update_time = "2014-12-18T09:17:03Z" - -[examples.Division.cartography] -prominence = 29 +license = "ODbL-1.0" +property = "" +record_id = "r166560@15" +update_time = "2025-08-16T14:25:55Z" [examples.Division.names] -primary = "Sia'atoutai" +primary = "Kauaʻi County" + +[examples.Division.names.common] +haw = "kalana Kauaʻi" [[examples.Division.names.rules]] +value = "Kauai County" variant = "alternate" -value = "Nafualu" [examples.Division.local_type] -en = "village" +en = "county" [[examples.DivisionArea]] -id = "eb9b112f-ec3c-47f7-b519-6f9f2e6fc2bd" -geometry = "MULTIPOLYGON (((-174.9553949 -21.4730179, -174.9514163 -21.4719978, -174.9520108 -21.4681253, -174.9566122 -21.4687535, -174.9553949 -21.4730179)), ((-174.9634398 -21.3476807, -174.9753507 -21.3833656, -174.9702168 -21.4037277, -174.950488 -21.4269887, -174.9082983 -21.4577763, -174.9004303 -21.4398142, -174.9048159 -21.3698688, -174.9165467 -21.3035402, -174.9126977 -21.2903268, -174.9199765 -21.2834922, -174.9634398 -21.3476807)))" -country = "TO" -version = 2 -subtype = "region" +admin_level = 2 class = "land" +country = "US" +division_id = "c9b8adc9-4639-4392-8efe-8401eeb19929" +geometry = "MULTIPOLYGON (((-160.5408313 21.6535414, -160.544491 21.6514764, -160.5369253 21.6495783, -160.5408313 21.6535414)), ((-160.0921364 22.005401, -160.1251483 21.9547066, -160.2260792 21.8900136, -160.2470031 21.841157, -160.2339532 21.7921394, -160.2022486 21.7794359, -160.1588705 21.864646, -160.0737812 21.8957962, -160.0826438 21.9302968, -160.05025 21.9848306, -160.0921364 22.005401)), ((-160.0881165 22.0243325, -160.1005193 22.0287469, -160.1026828 22.0166419, -160.0881165 22.0243325)), ((-159.7508422 21.9762242, -159.6679712 21.9531973, -159.604686 21.8923573, -159.5913126 21.9041661, -159.443465 21.8684403, -159.3461084 21.9373297, -159.3624768 21.952083, -159.3309486 21.9590284, -159.3357151 22.0450997, -159.2964497 22.1053479, -159.2926896 22.1434001, -159.335189 22.2040351, -159.4024533 22.2327054, -159.4282799 22.2164343, -159.4864746 22.2298148, -159.5060386 22.2031539, -159.5808829 22.2237075, -159.7228708 22.1497995, -159.7443872 22.099889, -159.7826601 22.0669155, -159.7849578 22.0149525, -159.7508422 21.9762242)))" +id = "109cfa53-bb13-4e37-aeb0-14f9a08c737d" is_land = true is_territorial = false -region = "TO-01" -division_id = "21597af0-b564-463c-a356-42c29e712b7d" +region = "US-HI" +subtype = "county" theme = "divisions" type = "division_area" +version = 7 [examples.DivisionArea.bbox] -xmin = -174.97535705566406 -xmax = -174.90040588378906 -ymin = -21.473018646240234 -ymax = -21.283489227294922 +xmax = -159.29266357421875 +xmin = -160.54449462890625 +ymax = 22.23270606994629 +ymin = 21.649578094482422 [[examples.DivisionArea.sources]] -property = "" dataset = "OpenStreetMap" -record_id = "r7247527@3" -update_time = "2020-12-30T18:41:56Z" +license = "ODbL-1.0" +property = "" +record_id = "r166560@15" +update_time = "2025-08-16T14:25:55Z" [examples.DivisionArea.names] -primary = "ʻEua" +primary = "Kauaʻi County" + +[examples.DivisionArea.names.common] +haw = "kalana Kauaʻi" + +[[examples.DivisionArea.names.rules]] +value = "Kauai County" +variant = "alternate" [[examples.DivisionBoundary]] -id = "2bdf68e4-860d-3d8c-a472-ccf439a5302a" -geometry = "LINESTRING (-147.064823 -15.4231537, -147.0519131 -15.2885069, -147.048482 -15.1511701)" -country = "PF" -version = 1 -subtype = "county" -class = "maritime" -is_land = false -is_territorial = true +admin_level = 2 +class = "land" +country = "US" division_ids = [ - "ae266459-63a4-4508-8295-0101e27d039b", - "d4a6873d-885a-4f2a-bc0f-37e9d9e874e4" + "a546a5ef-ba43-44ce-a7c3-1999b005b20f", + "0554d43c-de68-433c-9a6c-db58161c8b4a", ] +geometry = "LINESTRING (-157.0195397035514 21.183903480793326, -157.0201257 21.180663, -157.0084983 21.1786971, -157.0053877 21.1769627, -157.0060747 21.1738084, -157.0003054 21.1722773, -156.9809248 21.1746356, -156.9710241 21.172431, -156.9670723 21.1613703, -156.9550334 21.1531141, -156.9552315 21.1564667, -156.9607515 21.1621168, -156.9584334 21.162083, -156.9598705 21.1636745, -156.958371 21.1708964, -156.9602176 21.1762534, -156.9577173 21.1760764, -156.9556416 21.1715505, -156.9554893 21.157221, -156.9528625 21.1523977, -156.9442066 21.1504038, -156.9389218 21.1568278, -156.938089 21.1530661, -156.9348977 21.15157, -156.9283241 21.1419085, -156.9262384 21.1360023, -156.9186457 21.1279636, -156.9124605 21.1262614, -156.8975854 21.1285336, -156.8979892 21.1337635, -156.8958138 21.1376515, -156.8970853 21.1421243, -156.8929598 21.1451006, -156.8928438 21.1467406, -156.897116 21.1457141, -156.9036748 21.1485179, -156.9059125 21.1527496, -156.910528 21.1563906, -156.917332 21.1590391, -156.9140393 21.1691869)" +id = "321ed3ec-23c2-376e-a4fd-0df484cb99ca" is_disputed = false +is_land = true +is_territorial = false +region = "US-HI" +subtype = "county" theme = "divisions" type = "division_boundary" +version = 2 [examples.DivisionBoundary.bbox] -xmin = -147.06483459472656 -xmax = -147.04847717285156 -ymin = -15.4231538772583 -ymax = -15.151169776916504 +xmax = -156.89283752441406 +xmin = -157.02012634277344 +ymax = 21.18390655517578 +ymin = 21.12626075744629 [[examples.DivisionBoundary.sources]] -property = "" dataset = "OpenStreetMap" -record_id = "r6063055@9" -update_time = "2023-07-20T00:28:40Z" +license = "ODbL-1.0" +property = "" +record_id = "r166564@15" +update_time = "2025-02-21T16:35:23Z" [[examples.DivisionBoundary.sources]] -property = "" dataset = "OpenStreetMap" -record_id = "r6063063@12" -update_time = "2023-07-20T00:28:40Z" +license = "ODbL-1.0" +property = "" +record_id = "r166561@26" +update_time = "2025-08-16T14:01:24Z" diff --git a/packages/overture-schema-pyspark/README.md b/packages/overture-schema-pyspark/README.md new file mode 100644 index 000000000..fdadde3fc --- /dev/null +++ b/packages/overture-schema-pyspark/README.md @@ -0,0 +1,238 @@ +# overture-schema-pyspark + +PySpark validation expressions for Overture Maps data. Translates schema +constraints into composable PySpark Column expressions that validate +DataFrames and produce per-row, per-field error messages. + +Expression modules and the registry are generated by +[overture-schema-codegen](../overture-schema-codegen/). Regenerate after +schema changes rather than editing the generated output. + +## Usage + +### Python API + +```python +from pyspark.sql import SparkSession + +from overture.schema.pyspark import validate_model, explain_errors + +spark = SparkSession.builder.getOrCreate() +df = spark.read.parquet("samples/segment.parquet") + +result = validate_model(df, "segment") + +result.evaluated.cache() +total_rows = result.evaluated.count() +error_count = result.error_rows().count() +print(f"{error_count} / {total_rows} rows with errors") + +if error_count > 0: + violations = explain_errors(result.evaluated, result.checks) + violations.select("id", "field", "check", "message").show(truncate=False) +``` + +`validate_model()` looks up the feature type in the registry, compares +schemas, and evaluates all checks in a single pass. It returns a +`ValidationResult` with the evaluated DataFrame, the checks that ran, +any schema mismatches, and suppressed checks. + +| Function | Returns | Description | +| --- | --- | --- | +| `validate_model(df, model_type)` | `ValidationResult` | Registry lookup, schema comparison, check evaluation. | +| `result.error_rows()` | `DataFrame` | Rows with at least one violation. Original columns only. | +| `explain_errors(evaluated, checks)` | `DataFrame` | One row per violation. Adds `field`, `check`, `message` columns. | +| `model_names()` | `list[str]` | Available model type names, sorted. | + +Lower-level helpers (`evaluate_checks`, `filter_errors`) are available +for consumers needing finer control. All public symbols are re-exported +from `overture.schema.pyspark`. + +### CLI + +```bash +# Validate and show first 20 error rows (default) +overture-validate segment samples/segment.parquet + +# Custom output path, show first 50 violations +overture-validate segment samples/segment.parquet -o errors.parquet --head 50 + +# Count errors only (skip unpivot/explain) +overture-validate segment samples/segment.parquet --count-only + +# Pass Spark config +overture-validate segment samples/segment.parquet \ + --conf spark.master=local[4] + +# Continue past schema mismatches (e.g. Float vs Double on bbox) +overture-validate place s3a://overturemaps-us-west-2/release/2026-02-18.0 \ + --skip-schema-check + +# Skip checks for a column absent from the data +overture-validate segment data.parquet --skip-columns connector_ids + +# Ignore extra columns in the data that aren't in the schema +overture-validate segment data.parquet --ignore-extra-columns my_custom_col + +# Suppress all checks on a field +overture-validate segment data.parquet --suppress sources + +# Suppress a specific check (FIELD:CHECK) +overture-validate segment data.parquet --suppress version:bounds +``` + +The output Parquet contains one row per violation with the original columns +(minus geometry, if present) plus `field`, `check`, and `message`. Summary and the +first N violations print to the terminal; the full set is in the Parquet +file for further analysis. + +| Option | Description | +| --- | --- | +| `--skip-schema-check` | Warn on schema mismatches instead of aborting. | +| `--skip-columns COL` | Declare a column absent from the data; skips its checks and schema comparison. Repeatable. | +| `--ignore-extra-columns COL` | Ignore an extra data column not in the expected schema. Repeatable. | +| `--suppress SPEC` | Suppress checks. `FIELD` suppresses all checks on that root field; `FIELD:CHECK` suppresses one specific check. Repeatable. | +| `--count-only` | Report error count only; skip the explain/unpivot step. | +| `--conf KEY=VALUE` | Spark config pair. Repeatable. Overrides S3A defaults. | +| `-o`, `--output PATH` | Write violations to a Parquet file. | +| `--head N` | Number of error rows to display (default: 20). | + +### Path resolution + +The CLI resolves the input path to a Parquet read plan based on its +structure: + +| Path shape | Example | Behavior | +| --- | --- | --- | +| Hive partition path (contains `/theme=`) | `.../theme=transportation/type=segment/` | Reads directly; derives `basePath` so Spark discovers partition columns. | +| Individual file | `segment.parquet` | Reads directly; data already contains `theme`/`type` columns. | +| Release root | `s3a://overturemaps-us-west-2/release/2026-02-18.0` | Appends `theme={theme}/type={type}` using the schema's theme mapping; sets `basePath` to the original path. | + +This means you can point the CLI at a release root and it constructs the +full Hive path automatically: + +```bash +# These are equivalent: +overture-validate segment s3a://overturemaps-us-west-2/release/2026-02-18.0 +overture-validate segment s3a://overturemaps-us-west-2/release/2026-02-18.0/theme=transportation/type=segment/ +``` + +### Reading from S3 + +Paths starting with `s3a://` are detected automatically. The CLI +configures `hadoop-aws`, the S3A filesystem implementation, and +anonymous credentials -- no setup required for public buckets like +the Overture release bucket: + +```bash +overture-validate segment \ + s3a://overturemaps-us-west-2/release/2026-02-18.0/theme=transportation/type=segment/ +``` + +To use named AWS credentials instead of anonymous access: + +```bash +overture-validate segment \ + s3a://overturemaps-us-west-2/release/2026-02-18.0/theme=transportation/type=segment/ \ + --conf spark.hadoop.fs.s3a.aws.credentials.provider=software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider +``` + +Any `--conf` values override the S3A defaults. + +## Architecture + +```text +validate_model() Entry point -- registry lookup, schema check, evaluation + | + list[Check] Interface -- frozen (field, name, expr, shape) tuples + | + expression builders Translation -- schema constraints to Column expressions + (generated by registered in REGISTRY + overture-schema-codegen) + | + column_patterns / Reusable PySpark building blocks + constraint_expressions +``` + +**Check** is the interface between expression builders and composition. +Each `Check` carries a PySpark `Column` expression (unevaluated), a `field` +name for error grouping, a `name` identifying the check type (e.g. +`"required"`, `"bounds"`, `"enum"`), and a `shape` tag (`SCALAR` or `ARRAY`) +that tells `evaluate_checks()` how to normalize the result. + +Expression builders (like `connector_checks()`) are generated by +`overture-schema-codegen` from Pydantic schema models and registered in +`REGISTRY` by feature type name, paired with an expected `StructType` +schema via `ModelValidation`. + +## Generated expression builders + +Expression builders return `list[Check]`. The generated code uses constraint +expressions for common patterns and column patterns for structural wrappers. +Here's what the generated output looks like, using connector as an example: + +```python +from pyspark.sql import functions as F + +from overture.schema.pyspark.check import Check, CheckShape +from overture.schema.pyspark.expressions.column_patterns import array_check +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_bounds, + check_enum, + check_array_min_length, + check_required, +) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0), + shape=CheckShape.SCALAR, + root_field="version", + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["transportation"]), + shape=CheckShape.SCALAR, + root_field="theme", + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + root_field="sources", + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check( + "sources", + lambda el: check_required(el["dataset"]), + ), + shape=CheckShape.ARRAY, + root_field="sources", + ) +``` + +The registry maps feature type names to `ModelValidation` pairs: + +```python +from overture.schema.pyspark.check import ModelValidation +from overture.schema.pyspark._registry import REGISTRY + +# REGISTRY is auto-generated: +# REGISTRY["connector"] = ModelValidation(schema=CONNECTOR_SCHEMA, checks=connector_checks) +``` diff --git a/packages/overture-schema-pyspark/pyproject.toml b/packages/overture-schema-pyspark/pyproject.toml new file mode 100644 index 000000000..1206ee7ad --- /dev/null +++ b/packages/overture-schema-pyspark/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +build-backend = "hatchling.build" +requires = ["hatchling"] + +[project] +dependencies = [ + "click>=8.0", + "overture-schema-system", + "pyspark>=3.4", +] +description = "PySpark validation expressions for Overture Maps data" +dynamic = ["version"] +license = "MIT" +name = "overture-schema-pyspark" +requires-python = ">=3.10" + +[project.scripts] +overture-validate = "overture.schema.pyspark.cli:validate_cli" + +[tool.hatch.build.targets.wheel] +packages = ["src/overture"] + +[tool.hatch.version] +path = "src/overture/schema/pyspark/__about__.py" + +[tool.uv.sources] +overture-schema-system = { workspace = true } diff --git a/packages/overture-schema-pyspark/src/overture/__init__.py b/packages/overture-schema-pyspark/src/overture/__init__.py new file mode 100644 index 000000000..8db66d3d0 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/__init__.py @@ -0,0 +1 @@ +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/packages/overture-schema-pyspark/src/overture/schema/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/__init__.py new file mode 100644 index 000000000..8db66d3d0 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/__init__.py @@ -0,0 +1 @@ +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__about__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__about__.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__about__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py new file mode 100644 index 000000000..8841ae9f4 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py @@ -0,0 +1,27 @@ +"""PySpark validation expressions for Overture Maps data.""" + +from .check import Check, CheckShape +from .schema_check import SchemaMismatch, compare_schemas +from .validate import ( + ValidationResult, + evaluate_checks, + explain_errors, + filter_errors, + model_keys, + model_names, + validate_model, +) + +__all__ = [ + "Check", + "CheckShape", + "SchemaMismatch", + "ValidationResult", + "compare_schemas", + "evaluate_checks", + "explain_errors", + "model_keys", + "model_names", + "filter_errors", + "validate_model", +] diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/_registry.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/_registry.py new file mode 100644 index 000000000..dd40f92af --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/_registry.py @@ -0,0 +1,67 @@ +"""Runtime registry of feature validations. + +Built at import time by walking the generated `expressions.generated` +namespace and collecting every module that exposes the +codegen-emitted `ENTRY_POINT` and `MODEL_VALIDATION` constants. + +The generated tree on disk is the runtime source of truth: the +registry contains exactly what was generated, regardless of which +theme packages are installed alongside the pyspark package. A missing +`expressions/generated/` subtree simply yields an empty registry -- +the package still imports cleanly. +""" + +from __future__ import annotations + +import importlib +import logging +import pkgutil + +from .check import ModelValidation + +logger = logging.getLogger(__name__) + +_GENERATED_ROOT = "overture.schema.pyspark.expressions.generated" + + +def _walk() -> tuple[dict[str, ModelValidation], dict[str, dict[str, str]]]: + """Walk the generated tree and collect registry + partition map. + + Returns a `(registry, partition_map)` pair: + + * `registry` keys every feature by its `ENTRY_POINT` value. + * `partition_map` keys partitioned features by entry-point, mapping + to a Hive partition dict (e.g. `{"theme": "places", "type": + "place"}`) for path construction. Features with no `PARTITIONS` + data (empty dict) are omitted; the codegen only sets `PARTITIONS` + when the data lake organizes the feature by Hive partitions. + `type` is appended here from the module file name so consumers + get a complete partition path without the codegen having to + duplicate the type value. + """ + registry: dict[str, ModelValidation] = {} + partition_map: dict[str, dict[str, str]] = {} + + try: + root = importlib.import_module(_GENERATED_ROOT) + except ImportError: + return registry, partition_map + + for info in pkgutil.walk_packages(root.__path__, prefix=root.__name__ + "."): + if info.ispkg: + continue + module = importlib.import_module(info.name) + entry_point = getattr(module, "ENTRY_POINT", None) + validation = getattr(module, "MODEL_VALIDATION", None) + if entry_point is None or validation is None: + continue + registry[entry_point] = validation + partitions = getattr(module, "PARTITIONS", None) or {} + if partitions: + feature_type = info.name.rsplit(".", 1)[-1] + partition_map[entry_point] = {**partitions, "type": feature_type} + + return registry, partition_map + + +REGISTRY, PARTITION_MAP = _walk() diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/check.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/check.py new file mode 100644 index 000000000..9500cf4e7 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/check.py @@ -0,0 +1,51 @@ +"""Check dataclass — interface between expression builders and composition.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from enum import Enum + +from pyspark.sql import Column +from pyspark.sql.types import StructType + +from overture.schema.system.primitive import GeometryType + + +class CheckShape(Enum): + """How the composition layer handles a check expression.""" + + SCALAR = "scalar" # expression returns nullable string + ARRAY = "array" # expression returns array + + +@dataclass(frozen=True) +class Check: + """One validation check. + + `field` identifies what the check is about (for error column naming + and report grouping), not how to access the data. The expression in + `expr` already encodes the access pattern. + + `read_columns` names every top-level schema column the expression + dereferences -- one for a plain field check, several for a model-level + check that spans columns, plus any discriminator a variant gate reads. + `validate_model` drops a check when any column it reads is skipped or + structurally absent, so an unresolvable `F.col()` never reaches Spark; + it also treats these as the columns a check can be suppressed by name. + """ + + field: str + name: str + expr: Column + shape: CheckShape + read_columns: frozenset[str] + + +@dataclass(frozen=True) +class ModelValidation: + """Pairs an expected schema with check builders for a feature type.""" + + schema: StructType + checks: Callable[[], list[Check]] + geometry_types: tuple[GeometryType, ...] = () diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/cli.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/cli.py new file mode 100644 index 000000000..62d339128 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/cli.py @@ -0,0 +1,306 @@ +"""CLI entry point for validation.""" + +from __future__ import annotations + +import sys +from collections.abc import Collection, Mapping +from dataclasses import dataclass + +import click +from pyspark.errors import AnalysisException +from pyspark.sql import DataFrame, SparkSession + +from overture.schema.system.discovery import resolve_entry_point_key +from overture.schema.system.primitive import GeometryType + +from ._registry import PARTITION_MAP, REGISTRY +from .validate import ( + explain_errors, + model_names, + validate_model, +) + + +@dataclass(frozen=True) +class ReadSpec: + """Parquet read plan. + + `data_path` selects the files to read; `base_path`, when set, tells + Spark where to start discovering Hive partition columns. + """ + + data_path: str + base_path: str | None = None + + +def absent_column(exc: AnalysisException, columns: Collection[str]) -> str | None: + """The top-level column an unresolved-column error names, if absent from data. + + Returns the column name only when `exc` is an `UNRESOLVED_COLUMN` error + whose target is genuinely missing from `columns` -- the case a re-run with + `--skip-columns` resolves. Every other `AnalysisException` (a struct field + accessed on a scalar, a type mismatch, an unresolved column that is in fact + present) returns None, marking it a generator or expression bug to surface + rather than steer toward `--skip-columns`. + + Parameters + ---------- + exc + The exception raised while Spark planned the check expressions. + columns + The data's top-level column names (`df.columns`). + """ + condition = exc.getCondition() + if condition is None or not condition.startswith("UNRESOLVED_COLUMN"): + return None + object_name = (exc.getMessageParameters() or {}).get("objectName") + if not object_name: + return None + # objectName is backtick-quoted, e.g. `phantom` or `bbox`.`xmin`; the + # top-level segment is the column df.columns would carry. + top_level = object_name.split(".", 1)[0].strip("`") + return top_level if top_level not in columns else None + + +def resolve_read(path: str, partitions: Mapping[str, str] | None) -> ReadSpec: + """Determine read strategy from path structure. + + The partition map is an ordered Hive hierarchy + (`{"theme": "buildings", "type": "building"}`). A path supplies a + prefix of it; the leaves below the deepest level already present are + appended so the read always lands on a single feature type. Cases: + + 1. **Individual file** (`*.parquet`) or no partitions -- read + directly; data already contains the partition columns inline. + 2. **Release root** (no partition directories) -- append the full + partition path and set `basePath` to the original path. + 3. **Partial partition path** (`theme=X/`) -- append the missing + leaves (`type=Y`) so a single feature's checks aren't run against + every type sharing the theme directory. + 4. **Leaf partition path** (`theme=X/type=Y/`) -- nothing to append; + read it directly with `basePath` derived. + """ + stripped = path.rstrip("/") + + # Individual file or no partition mapping — data has partition columns inline + if stripped.endswith(".parquet") or not partitions: + return ReadSpec(data_path=path) + + keys = list(partitions) + # Partition levels already present in the path, in hierarchy order. + present = [i for i, key in enumerate(keys) if f"/{key}=" in stripped] + depth = present[-1] + 1 if present else 0 # count of levels already filled + leaves = "/".join(f"{key}={partitions[key]}" for key in keys[depth:]) + + if not present: + # Release root — append the full partition path; it is the base. + return ReadSpec(data_path=f"{stripped}/{leaves}", base_path=stripped) + + # Path already contains partition directories: the base is the release + # root (before the first one); append any leaves below the deepest + # present level (none for a leaf path, which then reads as-is). + base_idx = stripped.find(f"/{keys[present[0]]}=") + data_path = f"{stripped}/{leaves}" if leaves else path + return ReadSpec(data_path=data_path, base_path=stripped[:base_idx]) + + +def read_feature(spark: SparkSession, spec: ReadSpec) -> DataFrame: + """Read a DataFrame according to a ReadSpec.""" + reader = spark.read + if spec.base_path: + reader = reader.option("basePath", spec.base_path) + return reader.parquet(spec.data_path) + + +_S3A_DEFAULTS: dict[str, str] = { + "spark.jars.packages": "org.apache.hadoop:hadoop-aws:3.4.1", + "spark.hadoop.fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem", + "spark.hadoop.fs.s3a.aws.credentials.provider": ( + "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider" + ), +} + +_LARGE_GEOMETRY_TYPES = frozenset( + { + GeometryType.LINE_STRING, + GeometryType.MULTI_LINE_STRING, + GeometryType.POLYGON, + GeometryType.MULTI_POLYGON, + GeometryType.GEOMETRY_COLLECTION, + } +) + + +def _may_have_large_geometry(feature_key: str) -> bool: + """Whether a registered feature's geometries may be large. + + Returns True when the registered geometry types include + (multi)linestrings, (multi)polygons, or geometry collections, + or when geometry types are unspecified (safe default). + """ + validation = REGISTRY[feature_key] + if not validation.geometry_types: + return True + return bool(set(validation.geometry_types) & _LARGE_GEOMETRY_TYPES) + + +def _spark_config(path: str, conf: tuple[str, ...], feature_key: str) -> dict[str, str]: + """Build Spark config dict with safe defaults. + + Disables the vectorized Parquet reader for features with large + geometries (polygons, linestrings) to avoid OOM on WKB binary + columns. Adds S3A credentials for `s3a://` paths. User-supplied + `--conf` values override any defaults. + """ + config: dict[str, str] = {} + if _may_have_large_geometry(feature_key): + config["spark.sql.parquet.enableVectorizedReader"] = "false" + if path.startswith("s3a://"): + config.update(_S3A_DEFAULTS) + for pair in conf: + key, _, value = pair.partition("=") + config[key] = value + return config + + +@click.command("overture-validate") +@click.argument("feature_type") +@click.argument("path") +@click.option("-o", "--output", default=None, help="Output path for validated Parquet.") +@click.option( + "--head", + "head_n", + default=20, + type=int, + show_default=True, + help="Error rows to display.", +) +@click.option("--conf", multiple=True, help="Spark config key=value pairs.") +@click.option( + "--count-only", + is_flag=True, + default=False, + help="Report error count only; skip explain/unpivot.", +) +@click.option( + "--skip-schema-check", + is_flag=True, + default=False, + help="Warn on schema mismatches instead of aborting.", +) +@click.option( + "--skip-columns", + multiple=True, + help="Columns declared absent from data; skips their checks.", +) +@click.option( + "--ignore-extra-columns", + multiple=True, + help="Extra data columns to ignore in schema comparison.", +) +@click.option( + "--suppress", + "suppress_specs", + multiple=True, + help="Suppress checks: FIELD (all checks) or FIELD:CHECK (specific).", +) +def validate_cli( + feature_type: str, + path: str, + output: str | None, + head_n: int, + conf: tuple[str, ...], + count_only: bool, + skip_schema_check: bool, + skip_columns: tuple[str, ...], + ignore_extra_columns: tuple[str, ...], + suppress_specs: tuple[str, ...], +) -> None: + """Validate Overture data at PATH and write annotated Parquet.""" + try: + resolved = resolve_entry_point_key(feature_type, REGISTRY) + except ValueError: + click.echo( + f"Unknown type '{feature_type}'. Known: {', '.join(model_names())}", + err=True, + ) + sys.exit(1) + + builder = SparkSession.builder + for key, value in _spark_config(path, conf, resolved).items(): + builder = builder.config(key, value) + spark = builder.getOrCreate() + spark.sparkContext.setLogLevel("ERROR") + + spec = resolve_read(path, PARTITION_MAP.get(resolved)) + df = read_feature(spark, spec) + + suppress: list[str | tuple[str, str]] = [] + for s in suppress_specs: + if ":" in s: + field, name = s.split(":", 1) + suppress.append((field, name)) + else: + suppress.append(s) + + try: + result = validate_model( + df, + resolved, + skip_columns=skip_columns, + ignore_extra_columns=ignore_extra_columns, + suppress=suppress, + ) + except ValueError as e: + click.echo(str(e), err=True) + sys.exit(1) + except AnalysisException as e: + # Backstop, narrowed to the one cause `--skip-columns` can address: a + # check that names a column missing from the data. validate_model + # already drops checks for skipped and schema-absent columns, so this + # fires only on a column outside the expected schema -- offer the + # operator the skip lever and name the column. Every other + # AnalysisException (a type mismatch, a struct field read off a scalar) + # is a generator bug `--skip-columns` cannot fix; let it propagate as a + # traceback rather than mask it behind the skip hint. + column = absent_column(e, df.columns) + if column is None: + raise + click.echo( + f"A check references column '{column}', absent from the data at {path}.", + err=True, + ) + click.echo( + f"Re-run with `--skip-columns {column}` to skip its checks, " + "or `--skip-schema-check`.", + err=True, + ) + sys.exit(1) + + if result.schema_mismatches: + click.echo(f"Schema mismatches for {resolved}:", err=True) + for m in result.schema_mismatches: + click.echo(f" {m.path}: expected {m.expected}, got {m.actual}", err=True) + if result.absent_columns: + flags = " ".join(f"--skip-columns {c}" for c in result.absent_columns) + click.echo( + f" Re-run with `{flags}` to skip missing columns.", + err=True, + ) + if not skip_schema_check: + sys.exit(1) + + total_rows, error_count = result.row_counts() + click.echo(f"{error_count} / {total_rows} rows with errors", err=True) + + if error_count > 0: + if not count_only: + explained = explain_errors(result.evaluated, result.checks).drop("geometry") + if output and head_n > 0: + explained = explained.cache() + if output: + explained.write.mode("overwrite").parquet(output) + click.echo(f"Written to {output}", err=True) + if head_n > 0: + explained.show(head_n, truncate=False) + sys.exit(1) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/__init__.py new file mode 100644 index 000000000..572e57314 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/__init__.py @@ -0,0 +1 @@ +"""Expression builders and reusable PySpark column patterns.""" diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/_schema_structs.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/_schema_structs.py new file mode 100644 index 000000000..3bc8ea809 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/_schema_structs.py @@ -0,0 +1,22 @@ +"""Hand-written Spark StructType fragments for types the codegen can't generate. + +The codegen builds feature schemas by walking Pydantic `BaseModel` +subclasses. `BBox` is a plain class, not a `BaseModel`, so extraction +can't reach it -- `BBOX_STRUCT` is hand-written here to fill the gap. +Every other nested type is a `BaseModel` and gets generated directly +into each feature module, which is why this file holds only the one +struct. +""" + +from __future__ import annotations + +from pyspark.sql.types import DoubleType, StructField, StructType + +BBOX_STRUCT = StructType( + [ + StructField("xmin", DoubleType(), True), + StructField("xmax", DoubleType(), True), + StructField("ymin", DoubleType(), True), + StructField("ymax", DoubleType(), True), + ] +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/column_patterns.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/column_patterns.py new file mode 100644 index 000000000..18acbb8ef --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/column_patterns.py @@ -0,0 +1,141 @@ +"""Structural PySpark column patterns for validation expression composition. + +These functions provide reusable wrappers for array iteration, null +guarding, and error message construction. Expression builders and +constraint translators compose them; codegen calls them rather than +reimplementing the patterns. +""" + +from __future__ import annotations + +from collections.abc import Callable + +from pyspark.sql import Column +from pyspark.sql import functions as F + + +def error_msg(prefix: str, *value_cols: Column) -> Column: + """Build an error message: literal prefix followed by interpolated values. + + Each interpolated value is coalesced to a string before concatenation so + that a NULL value never makes the whole message NULL. `F.concat` returns + NULL if any argument is NULL, and a NULL message is silently dropped by + `array_compact` in the array-check path (or the scalar wrapper) -- which + would discard a real violation whenever the offending value is itself + NULL (e.g. a linear-reference range `[null, 1.5]`). + """ + safe = [F.coalesce(c.cast("string"), F.lit("null")) for c in value_cols] + return F.concat(F.lit(prefix), *safe) + + +def _resolve_column(column: str | Column) -> Column: + """Resolve a string column name to a Column, passing Column through.""" + return F.col(column) if isinstance(column, str) else column + + +def _null_guarded_transform( + col: Column, + check_fn: Callable[[Column], Column], + flatten: bool = False, +) -> Column: + """Null-guard, transform, optionally flatten, and compact. + + When `flatten=True`, null inner arrays are coalesced to empty before + flattening. `F.flatten` returns NULL whenever any inner array is + NULL, which would silently drop sibling errors -- inner `array_check` + legitimately returns NULL when its column is null (e.g. an optional + nested array that's absent on some elements but populated on others). + """ + transformed = F.transform(col, check_fn) + if flatten: + empty = F.array().cast("array") + transformed = F.flatten( + F.transform(transformed, lambda inner: F.coalesce(inner, empty)) + ) + return F.when(col.isNotNull(), F.array_compact(transformed)) + + +def array_check(column: str | Column, check_fn: Callable[[Column], Column]) -> Column: + """Null-guard a column, transform its elements, compact out nulls. + + *check_fn* receives each array element and returns a string Column + (error message) or null. + """ + return _null_guarded_transform(_resolve_column(column), check_fn) + + +def nested_array_check( + column: str | Column, check_fn: Callable[[Column], Column] +) -> Column: + """Like `array_check` but flattens nested error arrays. + + Use when *check_fn* itself returns an `array` (e.g. an + inner `array_check`). The outer transform produces + `array>`; this function flattens to `array` + before compacting nulls. + """ + return _null_guarded_transform(_resolve_column(column), check_fn, flatten=True) + + +def _map_projection_check( + column: str | Column, + projector: Callable[[Column], Column], + check_fn: Callable[[Column], Column], +) -> Column: + """Project a map column to an array, then null-guard and transform it. + + *projector* is `F.map_keys` or `F.map_values`. The projection already + yields a Column, so this calls `_null_guarded_transform` directly -- + routing through `array_check` would re-resolve an already-resolved + Column. A null map column projects to null, which the guard yields + through as null. + """ + return _null_guarded_transform(projector(_resolve_column(column)), check_fn) + + +def map_keys_check( + column: str | Column, check_fn: Callable[[Column], Column] +) -> Column: + """Validate a map's keys: project to `map_keys`, then array-check. + + *check_fn* receives each map key and returns a string Column (error + message) or null. A null map column yields null. + """ + return _map_projection_check(column, F.map_keys, check_fn) + + +def map_values_check( + column: str | Column, check_fn: Callable[[Column], Column] +) -> Column: + """Validate a map's values: project to `map_values`, then array-check. + + *check_fn* receives each map value and returns a string Column (error + message) or null. A null map column yields null. + """ + return _map_projection_check(column, F.map_values, check_fn) + + +def check_struct_unique(column: str | Column) -> Column: + """Check that an array has no duplicate items by whole-element comparison. + + Compares `size(col)` against `size(array_distinct(col))`. + `array_distinct` handles struct and nested-array elements natively + in Spark 3.4+. + + For string arrays (e.g. websites, socials), this compares raw values. + Pydantic's UniqueItemsConstraint on `list[HttpUrl]` compares + *normalized* URLs (adds trailing slash, lowercases host and scheme), + so it catches duplicates that differ only in normalization. This + check catches exact duplicates only — the difference is accepted. + """ + col = _resolve_column(column) + has_duplicates = F.size(col) > F.size(F.array_distinct(col)) + return F.when( + col.isNotNull(), + F.when(has_duplicates, F.lit("contains duplicate items")), + ) + + +def coalesce_errors(check: Column) -> Column: + """Wrap an array-producing check so nulls become empty arrays.""" + return F.coalesce(check, F.array().cast("array")) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/constraint_expressions.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/constraint_expressions.py new file mode 100644 index 000000000..52c422e2c --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/constraint_expressions.py @@ -0,0 +1,600 @@ +"""Constraint type to PySpark Column expression translation. + +Semantic translation layer: maps constraint parameters to Column +expressions that detect violations. Analogous to +`field_constraint_description.py` in overture-schema-codegen +(which maps constraints to prose). + +Each function takes a column accessor (`F.col("x")` or +`el["field"]`) and constraint parameters. Returns a Column that +evaluates to an error string on violation or null on success. Field +identity is carried structurally by `Check.field`, not embedded in +error messages. +""" + +from __future__ import annotations + +from collections.abc import Callable +from functools import reduce +from typing import Literal + +from pyspark.sql import Column +from pyspark.sql import functions as F + +from overture.schema.system.primitive import GeometryType + +from .column_patterns import error_msg + +_WKB_TYPE_CODE: dict[GeometryType, int] = { + GeometryType.POINT: 1, + GeometryType.LINE_STRING: 2, + GeometryType.POLYGON: 3, + GeometryType.MULTI_POINT: 4, + GeometryType.MULTI_LINE_STRING: 5, + GeometryType.MULTI_POLYGON: 6, + GeometryType.GEOMETRY_COLLECTION: 7, +} + +# A 3D/measured geometry encodes its extra dimensions in the WKB type word two +# different ways. EWKB (shapely's `.wkb` default, PostGIS, GDAL old-OGC) sets +# high flag bits -- Z=0x80000000, M=0x40000000, SRID=0x20000000 -- leaving the +# base type in the low bits. ISO WKB (mandated by GeoParquet) instead offsets +# the type by 1000/2000/3000 for Z/M/ZM (PointZ=1001). Masking off the EWKB +# flag nibble and then taking the value mod 1000 recovers the OGC base type +# (1-7) from either encoding. +_EWKB_FLAG_MASK = 0x0FFFFFFF +_ISO_DIMENSION_MODULUS = 1000 + +# A WKB header is a 1-byte order flag plus a 4-byte type word: 5 bytes, or 10 +# hex characters under F.hex. F.conv returns NULL only when its input is the +# empty string (a 0-1 byte blob), so a 2-4 byte blob parses a truncated header +# into a non-null bogus type and would pass; gate on hex length so every blob +# too short to carry a full type word is a violation. +_MIN_WKB_HEADER_HEX_LEN = 10 + + +_BOUND_OPS: dict[str, tuple[str, Callable[[Column, float | int], Column]]] = { + "ge": (">=", lambda c, v: c < v), + "gt": (">", lambda c, v: c <= v), + "le": ("<=", lambda c, v: c > v), + "lt": ("<", lambda c, v: c >= v), +} + + +def check_bounds( + col: Column, + *, + ge: float | int | None = None, + gt: float | int | None = None, + le: float | int | None = None, + lt: float | int | None = None, + check_nan: bool = True, +) -> Column: + """Numeric bounds check. Returns error string or null. + + Parameters + ---------- + col + Column to validate. + ge, gt, le, lt + Inclusive/exclusive lower and upper bounds. + check_nan + When True (default), prepend an explicit NaN guard so that NaN values + are rejected even on lower bounds (Spark sorts NaN above all values, + so a lower bound never fires on NaN without this guard). Set to False + for integer columns, where NaN is impossible and the guard is dead work. + """ + checks: list[Column] = [] + for key, value in (("ge", ge), ("gt", gt), ("le", le), ("lt", lt)): + if value is None: + continue + symbol, violates = _BOUND_OPS[key] + checks.append( + F.when( + violates(col, value), + error_msg( + f"must be {symbol} {value}, got ", + col.cast("string"), + ), + ) + ) + if not checks: + return F.lit(None).cast("string") + if not check_nan: + # null col -> every bound comparison is null, coalesce yields null + return F.coalesce(*checks) + # NaN satisfies no Pydantic bound (every comparison against it is False), + # but Spark sorts NaN above all values, so lower bounds (NaN < v / NaN <= v) + # never fire on it. Reject NaN explicitly whenever any bound applies. The + # cast keeps integer columns -- which can never be NaN -- from erroring. + nan_check = F.when( + F.isnan(col.cast("double")), + error_msg("must be a number, got ", col.cast("string")), + ) + # null col -> isnan is False (nan_check null) and every bound comparison is + # null, so coalesce yields null (no false positive) + return F.coalesce(nan_check, *checks) + + +def check_enum( + col: Column, + allowed: list[str], +) -> Column: + """Enum membership check. Returns error string or null.""" + return F.when( + col.isNotNull() & ~col.isin(allowed), + error_msg("invalid value '", col.cast("string"), F.lit("'")), + ) + + +def check_required(col: Column) -> Column: + """Null check for required fields. Returns error string or null.""" + return F.when(col.isNull(), F.lit("missing (null)")) + + +def check_pattern(col: Column, pattern: str, *, label: str) -> Column: + """Regex pattern check via rlike. Returns error string or null. + + Parameters + ---------- + col + Column to validate. + pattern + Java regex pattern (use `\\z` for absolute end-of-input). + label + Human-readable description used in error messages: + `"invalid {label}: got '...'"` + + Notes + ----- + `rlike` runs Java's regex engine against patterns authored for Python's + `re` (the engine Pydantic validates with). The dialects coincide on the + ASCII character ranges the schema patterns use, but diverge on the + shorthand classes: Java's `\\d \\s \\w \\S` are ASCII-only while Python's + are Unicode, so e.g. `^\\S+$` accepts a non-breaking space here that + Pydantic rejects, and `.` excludes a different set of line terminators. + These divergences are accepted -- the affected inputs (Unicode digits, + exotic whitespace) do not occur in practice for the constrained fields. + """ + msg = error_msg(f"invalid {label}: got '", col.cast("string"), F.lit("'")) + return F.when(col.isNotNull() & ~col.rlike(pattern), msg) + + +def except_literals(col: Column, error: Column, allowed: list[object]) -> Column: + """Suppress *error* when *col* equals one of the field's literal alternatives. + + A field typed `X | Literal[c, ...]` accepts the literals `c, ...` alongside + any value the concrete arm `X` validates. *error* is the concrete arm's + violation Column (error string or null). Returns null whenever *col* is one + of *allowed* (a permitted literal), and *error* unchanged otherwise. A null + *col* matches no literal and yields null -- the same result the wrapped + content checks already return for null, since presence is + `check_required`'s concern and is never wrapped here. + """ + return F.when(~col.isin(allowed), error) + + +def check_url_format(col: Column) -> Column: + """HTTP/HTTPS URL format check via pattern match. Returns error string or null. + + Pydantic's `HttpUrl` normalizes values (adds trailing slash, lowercases + host and scheme) before validation and comparison. This check validates + the raw string without normalization, with one concession to scheme + normalization: `(?i)` accepts an upper- or mixed-case scheme (`HTTP://`) + the way Pydantic's lowercasing does. Host case is left un-normalized, so + downstream uniqueness checks still compare un-normalized values. + """ + return check_pattern(col, r"(?i)^https?://[^\s]+\z", label="HTTP/HTTPS URL") + + +# Maximum HTTP(S) URL length, mirroring Pydantic's HttpUrl cap (the de facto +# 2083-character limit from legacy browsers). +_MAX_URL_LENGTH = 2083 + + +def check_url_length(col: Column) -> Column: + """URL length check: must not exceed the maximum URL length. Returns error string or null.""" + return F.when( + col.isNotNull() & (F.length(col) > _MAX_URL_LENGTH), + error_msg( + f"URL exceeds {_MAX_URL_LENGTH} characters: length ", + F.length(col).cast("string"), + ), + ) + + +def check_email(col: Column) -> Column: + """Email address format check. Returns error string or null.""" + return check_pattern( + col, + r"^[^\s@.]+(\.[^\s@.]+)*@([^\s@.]+\.)+[^\s@.]+\z", + label="email address", + ) + + +def _check_length( + col: Column, + measure: Column, + limit: int, + *, + direction: Literal["minimum", "maximum"], +) -> Column: + """Shared length-check logic for arrays and strings. + + *measure* is the pre-computed size/length column. + *direction* is `"minimum"` or `"maximum"`, controlling the + comparison operator and error label. + """ + violation = measure < limit if direction == "minimum" else measure > limit + return F.when( + col.isNotNull() & violation, + error_msg(f"{direction} length {limit}, got ", measure.cast("string")), + ) + + +def check_array_min_length(col: Column, min_len: int) -> Column: + """Array minimum length check. Returns error string or null.""" + return _check_length(col, F.size(col), min_len, direction="minimum") + + +def check_array_max_length(col: Column, max_len: int) -> Column: + """Array maximum length check. Returns error string or null.""" + return _check_length(col, F.size(col), max_len, direction="maximum") + + +def check_string_min_length(col: Column, min_len: int) -> Column: + """String minimum character length check. Returns error string or null.""" + return _check_length(col, F.length(col), min_len, direction="minimum") + + +def check_string_max_length(col: Column, max_len: int) -> Column: + """String maximum character length check. Returns error string or null.""" + return _check_length(col, F.length(col), max_len, direction="maximum") + + +_STRIPPED_PATTERN = r"(?sU)^[^\s\p{Cc}](.*[^\s\p{Cc}])?\z" +r"""Java regex: reject whitespace AND control characters at string boundaries. + +Boundary class `[^\s\p{Cc}]` rejects two categories at the first and +last character positions: + +1. **Whitespace** (`\s` with `(?U)`): Unicode `White_Space` property + — space, tab, newline, NBSP, em-space, etc. +2. **Control characters** (`\p{Cc}`): Unicode "Control" category — + C0 (U+0000-001F), DEL (U+007F), and C1 (U+0080-009F). + +Why both are needed: Python's `\s` (and `str.strip()`) treats +U+001C-001F (file/group/record/unit separators) as whitespace. Java's +`\s` with `(?U)` follows the Unicode `White_Space` property, which +excludes those four characters. Using `\S` alone in Java misses them, +allowing strings like `"Main St \x1f"` to pass. Adding `\p{Cc}` +closes that gap and also rejects other control characters (NUL, SOH, +DEL, C1 controls) that have no place at string boundaries. + +Interior control characters (middle of the string) are NOT rejected — +the `.*` in the middle position still matches anything. Policing +interior content is a separate concern. + +Divergence from Pydantic (accepted): Pydantic's stripped pattern +`^(\S(.*\S)?)?\Z` runs without DOTALL, so an interior newline makes it +fail (its `.` cannot cross the newline to reach the closing anchor). The +`(?s)` here lets `.*` cross newlines, so a string with an interior newline +but clean boundaries passes Spark while Pydantic rejects it. Stripped +fields are short identifiers/names where interior newlines do not occur, +so the looser behavior is accepted rather than matched. + +Flags: `(?s)` (DOTALL) lets `.*` cross newlines. `(?U)` +(UNICODE_CHARACTER_CLASS) gives `\s` full Unicode coverage. `\z` +(absolute end-of-input) avoids `$` matching before a trailing newline. +""" + + +def check_stripped(col: Column) -> Column: + """No leading/trailing whitespace or control characters. Returns error string or null.""" + return F.when( + col.isNotNull() & (F.length(col) > 0) & ~col.rlike(_STRIPPED_PATTERN), + error_msg("leading/trailing whitespace"), + ) + + +def check_json_pointer(col: Column) -> Column: + """JSON Pointer (RFC 6901) format check. + + Valid pointers start with `/` or are the empty string (which + references the whole document). + """ + return F.when( + col.isNotNull() & (col != "") & ~col.startswith("/"), + error_msg( + "invalid JSON pointer, must start with /, got '", + col.cast("string"), + F.lit("'"), + ), + ) + + +def check_linear_range_length(col: Column) -> Column: + """Linear reference range length check: exactly 2 elements.""" + size = F.size(col) + return F.when( + col.isNotNull() & (size != 2), + error_msg("must have exactly 2 elements, got ", size.cast("string")), + ) + + +def check_linear_range_bounds(col: Column) -> Column: + """Linear reference range bounds check: both values in [0.0, 1.0]. + + The `F.size(col) == 2` guard skips wrong-length arrays so this + check only fires when exactly two elements are present. Length + validation is `check_linear_range_length`'s responsibility. + """ + size = F.size(col) + v0, v1 = F.get(col, 0), F.get(col, 1) + return F.when( + col.isNotNull() + & (size == 2) + & ((v0 < 0.0) | (v0 > 1.0) | (v1 < 0.0) | (v1 > 1.0)), + error_msg( + "values must be in [0.0, 1.0], got [", + v0.cast("string"), + F.lit(", "), + v1.cast("string"), + F.lit("]"), + ), + ) + + +def check_linear_range_order(col: Column) -> Column: + """Linear reference range ordering check: start < end. + + The `F.size(col) == 2` guard skips wrong-length arrays so this + check only fires when exactly two elements are present. Length + validation is `check_linear_range_length`'s responsibility. + """ + size = F.size(col) + return F.when( + col.isNotNull() & (size == 2) & (F.get(col, 0) >= F.get(col, 1)), + error_msg("start must be < end"), + ) + + +def check_radio_group( + cols: list[Column], + field_names: list[str], +) -> Column: + """Exactly one of the given boolean columns must be True.""" + true_count = reduce( + lambda a, b: a + b, + (F.when(c, 1).otherwise(0) for c in cols), + ) + names = ", ".join(field_names) + return F.when( + true_count != 1, + error_msg( + f"exactly one of {names} must be true, got ", + true_count.cast("string"), + F.lit(" true"), + ), + ) + + +def _count_non_null(cols: list[Column]) -> Column: + """Sum of non-null indicators across *cols*.""" + return reduce( + lambda a, b: a + b, + (F.when(c.isNotNull(), 1).otherwise(0) for c in cols), + ) + + +def check_require_any_of( + cols: list[Column], + field_names: list[str], +) -> Column: + """At least one of the given columns must be non-null.""" + all_null = reduce(lambda a, b: a & b, (c.isNull() for c in cols)) + names = ", ".join(field_names) + return F.when(all_null, F.lit(f"requires at least one of {names}")) + + +def check_require_any_true( + conditions: list[Column], + field_names: list[str], +) -> Column: + """At least one of the given boolean conditions must be true. + + Each condition is a boolean Column (e.g. `is_land == True`). A null + condition counts as not-true -- mirroring Python's `None == True` -> + `False` -- so it is coalesced to `False` before the disjunction. The + check fires only when every condition is false or null. + """ + any_true = reduce( + lambda a, b: a | b, + (F.coalesce(c, F.lit(False)) for c in conditions), + ) + names = ", ".join(field_names) + return F.when(~any_true, F.lit(f"at least one of {names} must be true")) + + +def check_min_fields_set( + cols: list[Column], + field_names: list[str], + count: int, +) -> Column: + """At least *count* of the given columns must be non-null. + + Parameters + ---------- + cols + Column expressions to test for non-null. + field_names + Human-readable names for each column, used in the error message. + count + Minimum number of non-null columns required. + + Returns + ------- + Column + Error string on violation, null on success. + """ + non_null_count = _count_non_null(cols) + names = ", ".join(field_names) + return F.when( + non_null_count < count, + error_msg( + f"at least {count} of {names} required, got ", + non_null_count.cast("string"), + F.lit(" non-null"), + ), + ) + + +def _check_conditional_presence( + target: Column, + condition: Column, + condition_desc: str, + *condition_value_cols: Column, + expect_present: bool, +) -> Column: + """Shared logic for require_if / forbid_if. + + *expect_present=True* means target must be non-null when condition + holds (require); *False* means target must be null (forbid). + """ + word = "required" if expect_present else "forbidden" + target_test = target.isNull() if expect_present else target.isNotNull() + prefix = f"{word} when {condition_desc}" + if condition_value_cols: + interleaved = [ + p + for vc in condition_value_cols + for p in (F.lit(", got "), vc.cast("string")) + ] + msg = error_msg(prefix, *interleaved) + else: + msg = F.lit(prefix) + return F.when(condition & target_test, msg) + + +def check_require_if( + target: Column, + condition: Column, + condition_desc: str, + *condition_value_cols: Column, +) -> Column: + """Target must be non-null when condition is true.""" + return _check_conditional_presence( + target, + condition, + condition_desc, + *condition_value_cols, + expect_present=True, + ) + + +def check_forbid_if( + target: Column, + condition: Column, + condition_desc: str, + *condition_value_cols: Column, +) -> Column: + """Target must be null when condition is true.""" + return _check_conditional_presence( + target, + condition, + condition_desc, + *condition_value_cols, + expect_present=False, + ) + + +def check_geometry_type( + col: Column, + *allowed: GeometryType, +) -> Column: + """Geometry type check via WKB header parsing. + + Reads the endianness indicator and the 4-byte type word from the WKB + binary without deserializing coordinates. O(1) per row regardless of + geometry complexity. + + Normalizes both 3D/measured encodings (EWKB high flag bits and the ISO + WKB dimension offset) down to the OGC base type, so a valid PointZ + validates as a Point regardless of how its dimensions were encoded. + """ + hex_geom = F.hex(col) + byte_order = F.substring(hex_geom, 1, 2) + # The 4-byte type word follows the 1-byte order flag (hex positions 3-10). + # Big-endian stores it most-significant byte first (read as-is); little- + # endian stores it least-significant byte first, so reverse the byte pairs. + be_type_hex = F.substring(hex_geom, 3, 8) + le_type_hex = F.concat( + F.substring(hex_geom, 9, 2), + F.substring(hex_geom, 7, 2), + F.substring(hex_geom, 5, 2), + F.substring(hex_geom, 3, 2), + ) + type_word = F.conv( + F.when(byte_order == "01", le_type_hex).otherwise(be_type_hex), 16, 10 + ).cast("long") + base_type = type_word.bitwiseAND(_EWKB_FLAG_MASK) % _ISO_DIMENSION_MODULUS + allowed_codes = [_WKB_TYPE_CODE[t] for t in allowed] + names = " | ".join(t.geo_json_type for t in allowed) + if len(allowed_codes) == 1: + type_mismatch = base_type != allowed_codes[0] + else: + type_mismatch = ~base_type.isin(allowed_codes) + # A blob too short to hold the full 4-byte type word cannot be parsed; treat + # it as a violation rather than validating a bogus type read from a truncated + # header. The length gate subsumes the conv()-returns-NULL case (0-1 byte + # blob) and also catches the 2-4 byte blobs that parse to a non-null garbage + # type and would otherwise slip through. + violation = (F.length(hex_geom) < _MIN_WKB_HEADER_HEX_LEN) | type_mismatch + return F.when( + col.isNotNull() & violation, + error_msg(f"expected {names} geometry"), + ) + + +def check_bbox_completeness(col: Column) -> Column: + """Check that all bbox sub-fields are present when bbox is non-null.""" + return F.when( + col.isNotNull() + & ( + col["xmin"].isNull() + | col["ymin"].isNull() + | col["xmax"].isNull() + | col["ymax"].isNull() + ), + error_msg("bbox sub-fields must all be present"), + ) + + +def check_bbox_lat_ordering(col: Column) -> Column: + """Check that ymin does not exceed ymax.""" + return F.when( + col.isNotNull() & (col["ymin"] > col["ymax"]), + error_msg("expected ymin <= ymax"), + ) + + +def check_bbox_lat_range(col: Column) -> Column: + """Check that latitude values fall within [-90, 90].""" + return F.when( + col.isNotNull() + & ( + (col["ymin"] < -90) + | (col["ymin"] > 90) + | (col["ymax"] < -90) + | (col["ymax"] > 90) + ), + error_msg("latitude values must be in [-90, 90]"), + ) + + +# TODO: check_bbox_lon_ordering -- deferred pending antimeridian crossing +# policy. RFC 7946 section 5.2 allows xmin > xmax for bboxes that cross +# the antimeridian. + +# TODO: check_bbox_lon_range -- deferred pending decision on whether +# coordinates can wrap beyond [-180, 180]. diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/addresses/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/addresses/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/addresses/address.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/addresses/address.py new file mode 100644 index 000000000..58f8371cb --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/addresses/address.py @@ -0,0 +1,658 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Address validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + DoubleType, + IntegerType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_max_length, + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type(F.col("geometry"), GeometryType.POINT), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["addresses"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["address"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _address_levels_min_length_check() -> Check: + return Check( + field="address_levels_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("address_levels"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"address_levels"}), + ) + + +def _address_levels_max_length_check() -> Check: + return Check( + field="address_levels_max_length", + name="array_max_length", + expr=check_array_max_length(F.col("address_levels"), 5), + shape=CheckShape.SCALAR, + read_columns=frozenset({"address_levels"}), + ) + + +def _address_levels_value_string_min_length_check() -> Check: + return Check( + field="address_levels[].value", + name="string_min_length", + expr=array_check( + "address_levels", lambda el: check_string_min_length(el["value"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"address_levels"}), + ) + + +def _address_levels_value_stripped_check() -> Check: + return Check( + field="address_levels[].value", + name="stripped", + expr=array_check("address_levels", lambda el: check_stripped(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"address_levels"}), + ) + + +def _country_required_check() -> Check: + return Check( + field="country", + name="required", + expr=check_required(F.col("country")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"country"}), + ) + + +def _country_country_code_alpha2_check() -> Check: + return Check( + field="country", + name="country_code_alpha2", + expr=check_pattern( + F.col("country"), "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"country"}), + ) + + +def _number_string_min_length_check() -> Check: + return Check( + field="number", + name="string_min_length", + expr=check_string_min_length(F.col("number"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"number"}), + ) + + +def _number_stripped_check() -> Check: + return Check( + field="number", + name="stripped", + expr=check_stripped(F.col("number")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"number"}), + ) + + +def _postal_city_string_min_length_check() -> Check: + return Check( + field="postal_city", + name="string_min_length", + expr=check_string_min_length(F.col("postal_city"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"postal_city"}), + ) + + +def _postal_city_stripped_check() -> Check: + return Check( + field="postal_city", + name="stripped", + expr=check_stripped(F.col("postal_city")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"postal_city"}), + ) + + +def _postcode_string_min_length_check() -> Check: + return Check( + field="postcode", + name="string_min_length", + expr=check_string_min_length(F.col("postcode"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"postcode"}), + ) + + +def _postcode_stripped_check() -> Check: + return Check( + field="postcode", + name="stripped", + expr=check_stripped(F.col("postcode")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"postcode"}), + ) + + +def _street_string_min_length_check() -> Check: + return Check( + field="street", + name="string_min_length", + expr=check_string_min_length(F.col("street"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"street"}), + ) + + +def _street_stripped_check() -> Check: + return Check( + field="street", + name="stripped", + expr=check_stripped(F.col("street")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"street"}), + ) + + +def _unit_string_min_length_check() -> Check: + return Check( + field="unit", + name="string_min_length", + expr=check_string_min_length(F.col("unit"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"unit"}), + ) + + +def _unit_stripped_check() -> Check: + return Check( + field="unit", + name="stripped", + expr=check_stripped(F.col("unit")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"unit"}), + ) + + +def address_checks() -> list[Check]: + """All validation checks for address.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _address_levels_min_length_check(), + _address_levels_max_length_check(), + _address_levels_value_string_min_length_check(), + _address_levels_value_stripped_check(), + _country_required_check(), + _country_country_code_alpha2_check(), + _number_string_min_length_check(), + _number_stripped_check(), + _postal_city_string_min_length_check(), + _postal_city_stripped_check(), + _postcode_string_min_length_check(), + _postcode_stripped_check(), + _street_string_min_length_check(), + _street_stripped_check(), + _unit_string_min_length_check(), + _unit_stripped_check(), + ] + + +ADDRESS_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField( + "address_levels", + ArrayType(StructType([StructField("value", StringType(), True)]), True), + True, + ), + StructField("country", StringType(), True), + StructField("number", StringType(), True), + StructField("postal_city", StringType(), True), + StructField("postcode", StringType(), True), + StructField("street", StringType(), True), + StructField("unit", StringType(), True), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = (GeometryType.POINT,) + +ENTRY_POINT = "overture.schema.addresses:Address" + +PARTITIONS: dict[str, str] = {"theme": "addresses"} + +MODEL_VALIDATION = ModelValidation( + schema=ADDRESS_SCHEMA, + checks=address_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/annex/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/annex/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/annex/sources.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/annex/sources.py new file mode 100644 index 000000000..1ff387791 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/annex/sources.py @@ -0,0 +1,559 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Sources validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + DoubleType, + LongType, + MapType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + map_keys_check, + map_values_check, + nested_array_check, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_max_length, + check_array_min_length, + check_bounds, + check_enum, + check_pattern, + check_required, + check_url_format, + check_url_length, + except_literals, +) + + +def _datasets_check() -> Check: + return Check( + field="datasets", + name="required", + expr=check_required(F.col("datasets")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_source_name_check() -> Check: + return Check( + field="datasets[].source_name", + name="required", + expr=array_check("datasets", lambda el: check_required(el["source_name"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_source_dataset_name_check() -> Check: + return Check( + field="datasets[].source_dataset_name", + name="required", + expr=array_check( + "datasets", lambda el: check_required(el["source_dataset_name"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_data_url_required_check() -> Check: + return Check( + field="datasets[].data_url", + name="required", + expr=array_check("datasets", lambda el: check_required(el["data_url"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_data_url_url_format_check() -> Check: + return Check( + field="datasets[].data_url", + name="url_format", + expr=array_check( + "datasets", + lambda el: except_literals( + el["data_url"], check_url_format(el["data_url"]), [""] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_data_url_url_length_check() -> Check: + return Check( + field="datasets[].data_url", + name="url_length", + expr=array_check( + "datasets", + lambda el: except_literals( + el["data_url"], check_url_length(el["data_url"]), [""] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_data_url_archived_required_check() -> Check: + return Check( + field="datasets[].data_url_archived", + name="required", + expr=array_check( + "datasets", lambda el: check_required(el["data_url_archived"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_data_url_archived_url_format_check() -> Check: + return Check( + field="datasets[].data_url_archived", + name="url_format", + expr=array_check( + "datasets", + lambda el: except_literals( + el["data_url_archived"], check_url_format(el["data_url_archived"]), [""] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_data_url_archived_url_length_check() -> Check: + return Check( + field="datasets[].data_url_archived", + name="url_length", + expr=array_check( + "datasets", + lambda el: except_literals( + el["data_url_archived"], check_url_length(el["data_url_archived"]), [""] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_license_url_required_check() -> Check: + return Check( + field="datasets[].license_url", + name="required", + expr=array_check("datasets", lambda el: check_required(el["license_url"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_license_url_url_format_check() -> Check: + return Check( + field="datasets[].license_url", + name="url_format", + expr=array_check( + "datasets", + lambda el: except_literals( + el["license_url"], check_url_format(el["license_url"]), [""] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_license_url_url_length_check() -> Check: + return Check( + field="datasets[].license_url", + name="url_length", + expr=array_check( + "datasets", + lambda el: except_literals( + el["license_url"], check_url_length(el["license_url"]), [""] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_license_url_archived_required_check() -> Check: + return Check( + field="datasets[].license_url_archived", + name="required", + expr=array_check( + "datasets", lambda el: check_required(el["license_url_archived"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_license_url_archived_url_format_check() -> Check: + return Check( + field="datasets[].license_url_archived", + name="url_format", + expr=array_check( + "datasets", + lambda el: except_literals( + el["license_url_archived"], + check_url_format(el["license_url_archived"]), + [""], + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_license_url_archived_url_length_check() -> Check: + return Check( + field="datasets[].license_url_archived", + name="url_length", + expr=array_check( + "datasets", + lambda el: except_literals( + el["license_url_archived"], + check_url_length(el["license_url_archived"]), + [""], + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_license_type_check() -> Check: + return Check( + field="datasets[].license_type", + name="required", + expr=array_check("datasets", lambda el: check_required(el["license_type"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_license_text_check() -> Check: + return Check( + field="datasets[].license_text", + name="required", + expr=array_check("datasets", lambda el: check_required(el["license_text"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_license_attribution_check() -> Check: + return Check( + field="datasets[].license_attribution", + name="required", + expr=array_check( + "datasets", lambda el: check_required(el["license_attribution"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_coverage_bbox_check() -> Check: + return Check( + field="datasets[].coverage_bbox", + name="required", + expr=array_check("datasets", lambda el: check_required(el["coverage_bbox"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_coverage_bbox_min_length_check() -> Check: + return Check( + field="datasets[].coverage_bbox_min_length", + name="array_min_length", + expr=array_check( + "datasets", lambda el: check_array_min_length(el["coverage_bbox"], 4) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_coverage_bbox_max_length_check() -> Check: + return Check( + field="datasets[].coverage_bbox_max_length", + name="array_max_length", + expr=array_check( + "datasets", lambda el: check_array_max_length(el["coverage_bbox"], 4) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_url_url_format_check() -> Check: + return Check( + field="datasets[].url", + name="url_format", + expr=array_check("datasets", lambda el: check_url_format(el["url"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_url_url_length_check() -> Check: + return Check( + field="datasets[].url", + name="url_length", + expr=array_check("datasets", lambda el: check_url_length(el["url"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_url_archived_url_format_check() -> Check: + return Check( + field="datasets[].url_archived", + name="url_format", + expr=array_check("datasets", lambda el: check_url_format(el["url_archived"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_url_archived_url_length_check() -> Check: + return Check( + field="datasets[].url_archived", + name="url_length", + expr=array_check("datasets", lambda el: check_url_length(el["url_archived"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_data_download_url_url_format_check() -> Check: + return Check( + field="datasets[].data_download_url[]", + name="url_format", + expr=nested_array_check( + "datasets", + lambda el: array_check( + el["data_download_url"], + lambda inner: except_literals(inner, check_url_format(inner), [""]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_data_download_url_url_length_check() -> Check: + return Check( + field="datasets[].data_download_url[]", + name="url_length", + expr=nested_array_check( + "datasets", + lambda el: array_check( + el["data_download_url"], + lambda inner: except_literals(inner, check_url_length(inner), [""]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_countries_check() -> Check: + return Check( + field="datasets[].countries[]", + name="country_code_alpha2", + expr=nested_array_check( + "datasets", + lambda el: array_check( + el["countries"], + lambda inner: except_literals( + inner, + check_pattern( + inner, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ["Global"], + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_build_source_check() -> Check: + return Check( + field="datasets[].build_source", + name="enum", + expr=array_check( + "datasets", + lambda el: check_enum( + el["build_source"], ["OpenAddresses", "tf-data-platform"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _datasets_update_type_check() -> Check: + return Check( + field="datasets[].update_type", + name="enum", + expr=array_check( + "datasets", + lambda el: check_enum(el["update_type"], ["continuous", "manual"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"datasets"}), + ) + + +def _license_priority_check() -> Check: + return Check( + field="license_priority", + name="required", + expr=check_required(F.col("license_priority")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"license_priority"}), + ) + + +def _license_priority_key_check() -> Check: + return Check( + field="license_priority{key}", + name="pattern", + expr=map_keys_check( + "license_priority", + lambda k: check_pattern(k, "^[A-Za-z0-9._+\\-]+\\z", label="pattern"), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"license_priority"}), + ) + + +def _license_priority_value_check() -> Check: + return Check( + field="license_priority{value}", + name="bounds", + expr=map_values_check( + "license_priority", lambda v: check_bounds(v, ge=0, check_nan=False) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"license_priority"}), + ) + + +def sources_checks() -> list[Check]: + """All validation checks for sources.""" + return [ + _datasets_check(), + _datasets_source_name_check(), + _datasets_source_dataset_name_check(), + _datasets_data_url_required_check(), + _datasets_data_url_url_format_check(), + _datasets_data_url_url_length_check(), + _datasets_data_url_archived_required_check(), + _datasets_data_url_archived_url_format_check(), + _datasets_data_url_archived_url_length_check(), + _datasets_license_url_required_check(), + _datasets_license_url_url_format_check(), + _datasets_license_url_url_length_check(), + _datasets_license_url_archived_required_check(), + _datasets_license_url_archived_url_format_check(), + _datasets_license_url_archived_url_length_check(), + _datasets_license_type_check(), + _datasets_license_text_check(), + _datasets_license_attribution_check(), + _datasets_coverage_bbox_check(), + _datasets_coverage_bbox_min_length_check(), + _datasets_coverage_bbox_max_length_check(), + _datasets_url_url_format_check(), + _datasets_url_url_length_check(), + _datasets_url_archived_url_format_check(), + _datasets_url_archived_url_length_check(), + _datasets_data_download_url_url_format_check(), + _datasets_data_download_url_url_length_check(), + _datasets_countries_check(), + _datasets_build_source_check(), + _datasets_update_type_check(), + _license_priority_check(), + _license_priority_key_check(), + _license_priority_value_check(), + ] + + +SOURCES_SCHEMA = StructType( + [ + StructField( + "datasets", + ArrayType( + StructType( + [ + StructField("source_name", StringType(), True), + StructField("source_dataset_name", StringType(), True), + StructField("data_url", StringType(), True), + StructField("data_url_archived", StringType(), True), + StructField("license_url", StringType(), True), + StructField("license_url_archived", StringType(), True), + StructField("license_type", StringType(), True), + StructField("license_text", StringType(), True), + StructField("license_attribution", StringType(), True), + StructField( + "coverage_bbox", ArrayType(DoubleType(), True), True + ), + StructField("inception_date", StringType(), True), + StructField("url", StringType(), True), + StructField("url_archived", StringType(), True), + StructField( + "data_download_url", ArrayType(StringType(), True), True + ), + StructField("countries", ArrayType(StringType(), True), True), + StructField("coverage_description", StringType(), True), + StructField("data_layer_name", StringType(), True), + StructField("oa_path", ArrayType(StringType(), True), True), + StructField( + "address_levels", ArrayType(StringType(), True), True + ), + StructField("file_format", StringType(), True), + StructField("update_frequency", StringType(), True), + StructField("build_source", StringType(), True), + StructField("update_type", StringType(), True), + StructField( + "update_schedule", ArrayType(StringType(), True), True + ), + StructField("known_issues", StringType(), True), + StructField("notes", StringType(), True), + StructField("requires_attribution", StringType(), True), + ] + ), + True, + ), + True, + ), + StructField("license_priority", MapType(StringType(), LongType(), True), True), + ] +) + +ENTRY_POINT = "overture.schema.annex:Sources" + +PARTITIONS: dict[str, str] = {} + +MODEL_VALIDATION = ModelValidation( + schema=SOURCES_SCHEMA, + checks=sources_checks, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/bathymetry.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/bathymetry.py new file mode 100644 index 000000000..a4e3fe66f --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/bathymetry.py @@ -0,0 +1,572 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Bathymetry validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + DoubleType, + IntegerType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type( + F.col("geometry"), GeometryType.MULTI_POLYGON, GeometryType.POLYGON + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["base"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["bathymetry"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _depth_required_check() -> Check: + return Check( + field="depth", + name="required", + expr=check_required(F.col("depth")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"depth"}), + ) + + +def _depth_bounds_check() -> Check: + return Check( + field="depth", + name="bounds", + expr=check_bounds(F.col("depth"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"depth"}), + ) + + +def _cartography_prominence_bounds_check() -> Check: + return Check( + field="cartography.prominence_0", + name="bounds", + expr=check_bounds(F.col("cartography.prominence"), ge=1, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_prominence_bounds_check_1() -> Check: + return Check( + field="cartography.prominence_1", + name="bounds", + expr=check_bounds(F.col("cartography.prominence"), le=100, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_min_zoom_bounds_check() -> Check: + return Check( + field="cartography.min_zoom_0", + name="bounds", + expr=check_bounds(F.col("cartography.min_zoom"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_min_zoom_bounds_check_1() -> Check: + return Check( + field="cartography.min_zoom_1", + name="bounds", + expr=check_bounds(F.col("cartography.min_zoom"), le=23, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_max_zoom_bounds_check() -> Check: + return Check( + field="cartography.max_zoom_0", + name="bounds", + expr=check_bounds(F.col("cartography.max_zoom"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_max_zoom_bounds_check_1() -> Check: + return Check( + field="cartography.max_zoom_1", + name="bounds", + expr=check_bounds(F.col("cartography.max_zoom"), le=23, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def bathymetry_checks() -> list[Check]: + """All validation checks for bathymetry.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _depth_required_check(), + _depth_bounds_check(), + _cartography_prominence_bounds_check(), + _cartography_prominence_bounds_check_1(), + _cartography_min_zoom_bounds_check(), + _cartography_min_zoom_bounds_check_1(), + _cartography_max_zoom_bounds_check(), + _cartography_max_zoom_bounds_check_1(), + ] + + +BATHYMETRY_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("depth", IntegerType(), True), + StructField( + "cartography", + StructType( + [ + StructField("prominence", IntegerType(), True), + StructField("min_zoom", IntegerType(), True), + StructField("max_zoom", IntegerType(), True), + StructField("sort_key", IntegerType(), True), + ] + ), + True, + ), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = ( + GeometryType.MULTI_POLYGON, + GeometryType.POLYGON, +) + +ENTRY_POINT = "overture.schema.base:Bathymetry" + +PARTITIONS: dict[str, str] = {"theme": "base"} + +MODEL_VALIDATION = ModelValidation( + schema=BATHYMETRY_SCHEMA, + checks=bathymetry_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/infrastructure.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/infrastructure.py new file mode 100644 index 000000000..acd6725de --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/infrastructure.py @@ -0,0 +1,1122 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Infrastructure validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, + map_keys_check, + map_values_check, + nested_array_check, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type( + F.col("geometry"), + GeometryType.LINE_STRING, + GeometryType.MULTI_POLYGON, + GeometryType.POINT, + GeometryType.POLYGON, + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["base"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["infrastructure"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _class_required_check() -> Check: + return Check( + field="class", + name="required", + expr=check_required(F.col("class")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class"}), + ) + + +def _class_enum_check() -> Check: + return Check( + field="class", + name="enum", + expr=check_enum( + F.col("class"), + [ + "aerialway_station", + "airport", + "airport_gate", + "airstrip", + "apron", + "aqueduct", + "artwork", + "atm", + "barrier", + "bell_tower", + "bench", + "bicycle_parking", + "bicycle_rental", + "block", + "boardwalk", + "bollard", + "border_control", + "breakwater", + "bridge", + "bridge_support", + "bump_gate", + "bus_route", + "bus_station", + "bus_stop", + "bus_trap", + "cable", + "cable_barrier", + "cable_car", + "cable_distribution", + "camp_site", + "cantilever", + "catenary_mast", + "cattle_grid", + "chain", + "chair_lift", + "charging_station", + "city_wall", + "communication_line", + "communication_pole", + "communication_tower", + "connection", + "cooling", + "covered", + "crossing", + "cutline", + "cycle_barrier", + "dam", + "defensive", + "ditch", + "diving", + "drag_lift", + "drain", + "drinking_water", + "entrance", + "fence", + "ferry_terminal", + "fire_hydrant", + "fountain", + "full-height_turnstile", + "gasometer", + "gate", + "generator", + "give_way", + "gondola", + "goods", + "guard_rail", + "hampshire_gate", + "handrail", + "hedge", + "height_restrictor", + "heliostat", + "helipad", + "heliport", + "hose", + "information", + "insulator", + "international_airport", + "j-bar", + "jersey_barrier", + "kerb", + "kissing_gate", + "launchpad", + "lift_gate", + "lighting", + "lightning_protection", + "magic_carpet", + "manhole", + "milestone", + "military_airport", + "minaret", + "minor_line", + "mixed_lift", + "mobile_phone_tower", + "monitoring", + "motorcycle_parking", + "motorway_junction", + "movable", + "municipal_airport", + "observation", + "parking", + "parking_entrance", + "parking_space", + "pedestrian_crossing", + "picnic_table", + "pier", + "pipeline", + "plant", + "planter", + "platform", + "platter", + "portal", + "post_box", + "power_line", + "power_pole", + "power_tower", + "private_airport", + "pylon", + "quay", + "radar", + "railway_halt", + "railway_station", + "recycling", + "regional_airport", + "reservoir_covered", + "retaining_wall", + "roller_coaster", + "rope_tow", + "runway", + "sally_port", + "seaplane_airport", + "sewer", + "silo", + "siren", + "stile", + "stop", + "stop_position", + "stopway", + "storage_tank", + "street_cabinet", + "street_lamp", + "substation", + "subway_station", + "swing_gate", + "switch", + "t-bar", + "taxilane", + "taxiway", + "terminal", + "toilets", + "toll_booth", + "traffic_signals", + "transformer", + "trestle", + "utility_pole", + "vending_machine", + "viaduct", + "viewpoint", + "wall", + "waste_basket", + "waste_disposal", + "watchtower", + "water_tower", + "weir", + "zip_line", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class"}), + ) + + +def _subtype_required_check() -> Check: + return Check( + field="subtype", + name="required", + expr=check_required(F.col("subtype")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _subtype_enum_check() -> Check: + return Check( + field="subtype", + name="enum", + expr=check_enum( + F.col("subtype"), + [ + "aerialway", + "airport", + "barrier", + "bridge", + "communication", + "emergency", + "manhole", + "pedestrian", + "pier", + "power", + "quay", + "recreation", + "tower", + "transit", + "transportation", + "utility", + "waste_management", + "water", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _height_check() -> Check: + return Check( + field="height", + name="bounds", + expr=check_bounds(F.col("height"), gt=0.0), + shape=CheckShape.SCALAR, + read_columns=frozenset({"height"}), + ) + + +def _surface_check() -> Check: + return Check( + field="surface", + name="enum", + expr=check_enum( + F.col("surface"), + [ + "asphalt", + "cobblestone", + "compacted", + "concrete", + "concrete_plates", + "dirt", + "earth", + "fine_gravel", + "grass", + "gravel", + "ground", + "paved", + "paving_stones", + "pebblestone", + "recreation_grass", + "recreation_paved", + "recreation_sand", + "rubber", + "sand", + "sett", + "tartan", + "unpaved", + "wood", + "woodchips", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"surface"}), + ) + + +def _names_primary_required_check() -> Check: + return Check( + field="names.primary", + name="required", + expr=F.when(F.col("names").isNotNull(), check_required(F.col("names.primary"))), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_string_min_length_check() -> Check: + return Check( + field="names.primary", + name="string_min_length", + expr=check_string_min_length(F.col("names.primary"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_stripped_check() -> Check: + return Check( + field="names.primary", + name="stripped", + expr=check_stripped(F.col("names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_common_key_check() -> Check: + return Check( + field="names.common{key}", + name="language_tag", + expr=map_keys_check( + "names.common", + lambda k: check_pattern( + k, + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_common_value_check() -> Check: + return Check( + field="names.common{value}", + name="stripped", + expr=map_values_check("names.common", lambda v: check_stripped(v)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_required_check() -> Check: + return Check( + field="names.rules[].value", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_string_min_length_check() -> Check: + return Check( + field="names.rules[].value", + name="string_min_length", + expr=array_check( + "names.rules", lambda el: check_string_min_length(el["value"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_stripped_check() -> Check: + return Check( + field="names.rules[].value", + name="stripped", + expr=array_check("names.rules", lambda el: check_stripped(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_required_check() -> Check: + return Check( + field="names.rules[].variant", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["variant"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_enum_check() -> Check: + return Check( + field="names.rules[].variant", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["variant"], ["common", "official", "alternate", "short"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_language_check() -> Check: + return Check( + field="names.rules[].language", + name="language_tag", + expr=array_check( + "names.rules", + lambda el: check_pattern( + el["language"], + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_required_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["mode"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_enum_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["perspectives"]["mode"], ["accepted_by", "disputed_by"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check() -> Check: + return Check( + field="names.rules[].perspectives.countries", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["countries"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_min_length_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_min_length", + name="array_min_length", + expr=array_check( + "names.rules", + lambda el: check_array_min_length(el["perspectives"]["countries"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_unique_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_unique", + name="struct_unique", + expr=array_check( + "names.rules", + lambda el: check_struct_unique(el["perspectives"]["countries"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check_1() -> Check: + return Check( + field="names.rules[].perspectives.countries[]", + name="country_code_alpha2", + expr=nested_array_check( + "names.rules", + lambda el: array_check( + el["perspectives"]["countries"], + lambda inner: check_pattern( + inner, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_length_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_length", + expr=array_check( + "names.rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_bounds", + expr=array_check( + "names.rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_order_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_order", + expr=array_check( + "names.rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_side_check() -> Check: + return Check( + field="names.rules[].side", + name="enum", + expr=array_check( + "names.rules", lambda el: check_enum(el["side"], ["left", "right"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _wikidata_check() -> Check: + return Check( + field="wikidata", + name="wikidata_id", + expr=check_pattern( + F.col("wikidata"), + "^Q\\d+\\z", + label="Wikidata identifier (Q followed by digits)", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"wikidata"}), + ) + + +def infrastructure_checks() -> list[Check]: + """All validation checks for infrastructure.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _class_required_check(), + _class_enum_check(), + _subtype_required_check(), + _subtype_enum_check(), + _height_check(), + _surface_check(), + _names_primary_required_check(), + _names_primary_string_min_length_check(), + _names_primary_stripped_check(), + _names_common_key_check(), + _names_common_value_check(), + _names_rules_value_required_check(), + _names_rules_value_string_min_length_check(), + _names_rules_value_stripped_check(), + _names_rules_variant_required_check(), + _names_rules_variant_enum_check(), + _names_rules_language_check(), + _names_rules_perspectives_mode_required_check(), + _names_rules_perspectives_mode_enum_check(), + _names_rules_perspectives_countries_check(), + _names_rules_perspectives_countries_min_length_check(), + _names_rules_perspectives_countries_unique_check(), + _names_rules_perspectives_countries_check_1(), + _names_rules_between_linear_range_length_check(), + _names_rules_between_linear_range_bounds_check(), + _names_rules_between_linear_range_order_check(), + _names_rules_side_check(), + _wikidata_check(), + ] + + +INFRASTRUCTURE_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("class", StringType(), True), + StructField("subtype", StringType(), True), + StructField("height", DoubleType(), True), + StructField("surface", StringType(), True), + StructField( + "names", + StructType( + [ + StructField("primary", StringType(), True), + StructField( + "common", MapType(StringType(), StringType(), True), True + ), + StructField( + "rules", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("variant", StringType(), True), + StructField("language", StringType(), True), + StructField( + "perspectives", + StructType( + [ + StructField("mode", StringType(), True), + StructField( + "countries", + ArrayType(StringType(), True), + True, + ), + ] + ), + True, + ), + StructField( + "between", ArrayType(DoubleType(), True), True + ), + StructField("side", StringType(), True), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + StructField("level", IntegerType(), True), + StructField("source_tags", MapType(StringType(), StringType(), True), True), + StructField("wikidata", StringType(), True), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = ( + GeometryType.LINE_STRING, + GeometryType.MULTI_POLYGON, + GeometryType.POINT, + GeometryType.POLYGON, +) + +ENTRY_POINT = "overture.schema.base:Infrastructure" + +PARTITIONS: dict[str, str] = {"theme": "base"} + +MODEL_VALIDATION = ModelValidation( + schema=INFRASTRUCTURE_SCHEMA, + checks=infrastructure_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/land.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/land.py new file mode 100644 index 000000000..b3ab169fb --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/land.py @@ -0,0 +1,973 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Land validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, + map_keys_check, + map_values_check, + nested_array_check, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type( + F.col("geometry"), + GeometryType.LINE_STRING, + GeometryType.MULTI_POLYGON, + GeometryType.POINT, + GeometryType.POLYGON, + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["base"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["land"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _class_check() -> Check: + return Check( + field="class", + name="enum", + expr=check_enum( + F.col("class"), + [ + "archipelago", + "bare_rock", + "beach", + "cave_entrance", + "cliff", + "desert", + "dune", + "fell", + "forest", + "glacier", + "grass", + "grassland", + "heath", + "hill", + "island", + "islet", + "land", + "meadow", + "meteor_crater", + "mountain_range", + "peak", + "peninsula", + "plateau", + "reef", + "ridge", + "rock", + "saddle", + "sand", + "scree", + "scrub", + "shingle", + "shrub", + "shrubbery", + "stone", + "tree", + "tree_row", + "tundra", + "valley", + "volcanic_caldera_rim", + "volcano", + "wetland", + "wood", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class"}), + ) + + +def _subtype_check() -> Check: + return Check( + field="subtype", + name="enum", + expr=check_enum( + F.col("subtype"), + [ + "crater", + "desert", + "forest", + "glacier", + "grass", + "land", + "physical", + "reef", + "rock", + "sand", + "shrub", + "tree", + "wetland", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _elevation_check() -> Check: + return Check( + field="elevation", + name="bounds", + expr=check_bounds(F.col("elevation"), le=9000, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"elevation"}), + ) + + +def _surface_check() -> Check: + return Check( + field="surface", + name="enum", + expr=check_enum( + F.col("surface"), + [ + "asphalt", + "cobblestone", + "compacted", + "concrete", + "concrete_plates", + "dirt", + "earth", + "fine_gravel", + "grass", + "gravel", + "ground", + "paved", + "paving_stones", + "pebblestone", + "recreation_grass", + "recreation_paved", + "recreation_sand", + "rubber", + "sand", + "sett", + "tartan", + "unpaved", + "wood", + "woodchips", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"surface"}), + ) + + +def _names_primary_required_check() -> Check: + return Check( + field="names.primary", + name="required", + expr=F.when(F.col("names").isNotNull(), check_required(F.col("names.primary"))), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_string_min_length_check() -> Check: + return Check( + field="names.primary", + name="string_min_length", + expr=check_string_min_length(F.col("names.primary"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_stripped_check() -> Check: + return Check( + field="names.primary", + name="stripped", + expr=check_stripped(F.col("names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_common_key_check() -> Check: + return Check( + field="names.common{key}", + name="language_tag", + expr=map_keys_check( + "names.common", + lambda k: check_pattern( + k, + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_common_value_check() -> Check: + return Check( + field="names.common{value}", + name="stripped", + expr=map_values_check("names.common", lambda v: check_stripped(v)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_required_check() -> Check: + return Check( + field="names.rules[].value", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_string_min_length_check() -> Check: + return Check( + field="names.rules[].value", + name="string_min_length", + expr=array_check( + "names.rules", lambda el: check_string_min_length(el["value"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_stripped_check() -> Check: + return Check( + field="names.rules[].value", + name="stripped", + expr=array_check("names.rules", lambda el: check_stripped(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_required_check() -> Check: + return Check( + field="names.rules[].variant", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["variant"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_enum_check() -> Check: + return Check( + field="names.rules[].variant", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["variant"], ["common", "official", "alternate", "short"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_language_check() -> Check: + return Check( + field="names.rules[].language", + name="language_tag", + expr=array_check( + "names.rules", + lambda el: check_pattern( + el["language"], + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_required_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["mode"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_enum_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["perspectives"]["mode"], ["accepted_by", "disputed_by"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check() -> Check: + return Check( + field="names.rules[].perspectives.countries", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["countries"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_min_length_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_min_length", + name="array_min_length", + expr=array_check( + "names.rules", + lambda el: check_array_min_length(el["perspectives"]["countries"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_unique_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_unique", + name="struct_unique", + expr=array_check( + "names.rules", + lambda el: check_struct_unique(el["perspectives"]["countries"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check_1() -> Check: + return Check( + field="names.rules[].perspectives.countries[]", + name="country_code_alpha2", + expr=nested_array_check( + "names.rules", + lambda el: array_check( + el["perspectives"]["countries"], + lambda inner: check_pattern( + inner, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_length_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_length", + expr=array_check( + "names.rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_bounds", + expr=array_check( + "names.rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_order_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_order", + expr=array_check( + "names.rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_side_check() -> Check: + return Check( + field="names.rules[].side", + name="enum", + expr=array_check( + "names.rules", lambda el: check_enum(el["side"], ["left", "right"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _wikidata_check() -> Check: + return Check( + field="wikidata", + name="wikidata_id", + expr=check_pattern( + F.col("wikidata"), + "^Q\\d+\\z", + label="Wikidata identifier (Q followed by digits)", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"wikidata"}), + ) + + +def land_checks() -> list[Check]: + """All validation checks for land.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _class_check(), + _subtype_check(), + _elevation_check(), + _surface_check(), + _names_primary_required_check(), + _names_primary_string_min_length_check(), + _names_primary_stripped_check(), + _names_common_key_check(), + _names_common_value_check(), + _names_rules_value_required_check(), + _names_rules_value_string_min_length_check(), + _names_rules_value_stripped_check(), + _names_rules_variant_required_check(), + _names_rules_variant_enum_check(), + _names_rules_language_check(), + _names_rules_perspectives_mode_required_check(), + _names_rules_perspectives_mode_enum_check(), + _names_rules_perspectives_countries_check(), + _names_rules_perspectives_countries_min_length_check(), + _names_rules_perspectives_countries_unique_check(), + _names_rules_perspectives_countries_check_1(), + _names_rules_between_linear_range_length_check(), + _names_rules_between_linear_range_bounds_check(), + _names_rules_between_linear_range_order_check(), + _names_rules_side_check(), + _wikidata_check(), + ] + + +LAND_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("class", StringType(), True), + StructField("subtype", StringType(), True), + StructField("elevation", IntegerType(), True), + StructField("surface", StringType(), True), + StructField( + "names", + StructType( + [ + StructField("primary", StringType(), True), + StructField( + "common", MapType(StringType(), StringType(), True), True + ), + StructField( + "rules", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("variant", StringType(), True), + StructField("language", StringType(), True), + StructField( + "perspectives", + StructType( + [ + StructField("mode", StringType(), True), + StructField( + "countries", + ArrayType(StringType(), True), + True, + ), + ] + ), + True, + ), + StructField( + "between", ArrayType(DoubleType(), True), True + ), + StructField("side", StringType(), True), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + StructField("level", IntegerType(), True), + StructField("source_tags", MapType(StringType(), StringType(), True), True), + StructField("wikidata", StringType(), True), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = ( + GeometryType.LINE_STRING, + GeometryType.MULTI_POLYGON, + GeometryType.POINT, + GeometryType.POLYGON, +) + +ENTRY_POINT = "overture.schema.base:Land" + +PARTITIONS: dict[str, str] = {"theme": "base"} + +MODEL_VALIDATION = ModelValidation( + schema=LAND_SCHEMA, + checks=land_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/land_cover.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/land_cover.py new file mode 100644 index 000000000..d31c7596d --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/land_cover.py @@ -0,0 +1,586 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Land Cover validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + DoubleType, + IntegerType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type( + F.col("geometry"), GeometryType.MULTI_POLYGON, GeometryType.POLYGON + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["base"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["land_cover"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _subtype_required_check() -> Check: + return Check( + field="subtype", + name="required", + expr=check_required(F.col("subtype")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _subtype_enum_check() -> Check: + return Check( + field="subtype", + name="enum", + expr=check_enum( + F.col("subtype"), + [ + "barren", + "crop", + "forest", + "grass", + "mangrove", + "moss", + "shrub", + "snow", + "urban", + "wetland", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _cartography_prominence_bounds_check() -> Check: + return Check( + field="cartography.prominence_0", + name="bounds", + expr=check_bounds(F.col("cartography.prominence"), ge=1, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_prominence_bounds_check_1() -> Check: + return Check( + field="cartography.prominence_1", + name="bounds", + expr=check_bounds(F.col("cartography.prominence"), le=100, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_min_zoom_bounds_check() -> Check: + return Check( + field="cartography.min_zoom_0", + name="bounds", + expr=check_bounds(F.col("cartography.min_zoom"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_min_zoom_bounds_check_1() -> Check: + return Check( + field="cartography.min_zoom_1", + name="bounds", + expr=check_bounds(F.col("cartography.min_zoom"), le=23, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_max_zoom_bounds_check() -> Check: + return Check( + field="cartography.max_zoom_0", + name="bounds", + expr=check_bounds(F.col("cartography.max_zoom"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_max_zoom_bounds_check_1() -> Check: + return Check( + field="cartography.max_zoom_1", + name="bounds", + expr=check_bounds(F.col("cartography.max_zoom"), le=23, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def land_cover_checks() -> list[Check]: + """All validation checks for land_cover.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _subtype_required_check(), + _subtype_enum_check(), + _cartography_prominence_bounds_check(), + _cartography_prominence_bounds_check_1(), + _cartography_min_zoom_bounds_check(), + _cartography_min_zoom_bounds_check_1(), + _cartography_max_zoom_bounds_check(), + _cartography_max_zoom_bounds_check_1(), + ] + + +LAND_COVER_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("subtype", StringType(), True), + StructField( + "cartography", + StructType( + [ + StructField("prominence", IntegerType(), True), + StructField("min_zoom", IntegerType(), True), + StructField("max_zoom", IntegerType(), True), + StructField("sort_key", IntegerType(), True), + ] + ), + True, + ), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = ( + GeometryType.MULTI_POLYGON, + GeometryType.POLYGON, +) + +ENTRY_POINT = "overture.schema.base:LandCover" + +PARTITIONS: dict[str, str] = {"theme": "base"} + +MODEL_VALIDATION = ModelValidation( + schema=LAND_COVER_SCHEMA, + checks=land_cover_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/land_use.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/land_use.py new file mode 100644 index 000000000..5d894ccfd --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/land_use.py @@ -0,0 +1,1073 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Land Use validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, + map_keys_check, + map_values_check, + nested_array_check, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type( + F.col("geometry"), + GeometryType.LINE_STRING, + GeometryType.MULTI_POLYGON, + GeometryType.POINT, + GeometryType.POLYGON, + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["base"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["land_use"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _class_required_check() -> Check: + return Check( + field="class", + name="required", + expr=check_required(F.col("class")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class"}), + ) + + +def _class_enum_check() -> Check: + return Check( + field="class", + name="enum", + expr=check_enum( + F.col("class"), + [ + "aboriginal_land", + "airfield", + "allotments", + "animal_keeping", + "aquaculture", + "barracks", + "base", + "beach_resort", + "brownfield", + "bunker", + "camp_site", + "cemetery", + "clinic", + "college", + "commercial", + "connection", + "construction", + "danger_area", + "doctors", + "dog_park", + "downhill", + "driving_range", + "driving_school", + "education", + "environmental", + "fairway", + "farmland", + "farmyard", + "fatbike", + "flowerbed", + "forest", + "garages", + "garden", + "golf_course", + "grass", + "grave_yard", + "green", + "greenfield", + "greenhouse_horticulture", + "highway", + "hike", + "hospital", + "ice_skate", + "industrial", + "institutional", + "kindergarten", + "landfill", + "lateral_water_hazard", + "logging", + "marina", + "meadow", + "military", + "military_hospital", + "military_school", + "music_school", + "national_park", + "natural_monument", + "nature_reserve", + "naval_base", + "nordic", + "nuclear_explosion_site", + "obstacle_course", + "orchard", + "park", + "peat_cutting", + "pedestrian", + "pitch", + "plant_nursery", + "playground", + "plaza", + "protected", + "protected_landscape_seascape", + "quarry", + "railway", + "range", + "recreation_ground", + "religious", + "residential", + "resort", + "retail", + "rough", + "salt_pond", + "school", + "schoolyard", + "ski_jump", + "skitour", + "sled", + "sleigh", + "snow_park", + "species_management_area", + "stadium", + "state_park", + "static_caravan", + "strict_nature_reserve", + "tee", + "theme_park", + "track", + "traffic_island", + "training_area", + "trench", + "university", + "village_green", + "vineyard", + "water_hazard", + "water_park", + "wilderness_area", + "winter_sports", + "works", + "zoo", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class"}), + ) + + +def _subtype_required_check() -> Check: + return Check( + field="subtype", + name="required", + expr=check_required(F.col("subtype")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _subtype_enum_check() -> Check: + return Check( + field="subtype", + name="enum", + expr=check_enum( + F.col("subtype"), + [ + "agriculture", + "aquaculture", + "campground", + "cemetery", + "construction", + "developed", + "education", + "entertainment", + "golf", + "grass", + "horticulture", + "landfill", + "managed", + "medical", + "military", + "park", + "pedestrian", + "protected", + "recreation", + "religious", + "residential", + "resource_extraction", + "transportation", + "winter_sports", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _elevation_check() -> Check: + return Check( + field="elevation", + name="bounds", + expr=check_bounds(F.col("elevation"), le=9000, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"elevation"}), + ) + + +def _surface_check() -> Check: + return Check( + field="surface", + name="enum", + expr=check_enum( + F.col("surface"), + [ + "asphalt", + "cobblestone", + "compacted", + "concrete", + "concrete_plates", + "dirt", + "earth", + "fine_gravel", + "grass", + "gravel", + "ground", + "paved", + "paving_stones", + "pebblestone", + "recreation_grass", + "recreation_paved", + "recreation_sand", + "rubber", + "sand", + "sett", + "tartan", + "unpaved", + "wood", + "woodchips", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"surface"}), + ) + + +def _names_primary_required_check() -> Check: + return Check( + field="names.primary", + name="required", + expr=F.when(F.col("names").isNotNull(), check_required(F.col("names.primary"))), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_string_min_length_check() -> Check: + return Check( + field="names.primary", + name="string_min_length", + expr=check_string_min_length(F.col("names.primary"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_stripped_check() -> Check: + return Check( + field="names.primary", + name="stripped", + expr=check_stripped(F.col("names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_common_key_check() -> Check: + return Check( + field="names.common{key}", + name="language_tag", + expr=map_keys_check( + "names.common", + lambda k: check_pattern( + k, + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_common_value_check() -> Check: + return Check( + field="names.common{value}", + name="stripped", + expr=map_values_check("names.common", lambda v: check_stripped(v)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_required_check() -> Check: + return Check( + field="names.rules[].value", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_string_min_length_check() -> Check: + return Check( + field="names.rules[].value", + name="string_min_length", + expr=array_check( + "names.rules", lambda el: check_string_min_length(el["value"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_stripped_check() -> Check: + return Check( + field="names.rules[].value", + name="stripped", + expr=array_check("names.rules", lambda el: check_stripped(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_required_check() -> Check: + return Check( + field="names.rules[].variant", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["variant"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_enum_check() -> Check: + return Check( + field="names.rules[].variant", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["variant"], ["common", "official", "alternate", "short"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_language_check() -> Check: + return Check( + field="names.rules[].language", + name="language_tag", + expr=array_check( + "names.rules", + lambda el: check_pattern( + el["language"], + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_required_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["mode"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_enum_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["perspectives"]["mode"], ["accepted_by", "disputed_by"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check() -> Check: + return Check( + field="names.rules[].perspectives.countries", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["countries"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_min_length_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_min_length", + name="array_min_length", + expr=array_check( + "names.rules", + lambda el: check_array_min_length(el["perspectives"]["countries"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_unique_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_unique", + name="struct_unique", + expr=array_check( + "names.rules", + lambda el: check_struct_unique(el["perspectives"]["countries"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check_1() -> Check: + return Check( + field="names.rules[].perspectives.countries[]", + name="country_code_alpha2", + expr=nested_array_check( + "names.rules", + lambda el: array_check( + el["perspectives"]["countries"], + lambda inner: check_pattern( + inner, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_length_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_length", + expr=array_check( + "names.rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_bounds", + expr=array_check( + "names.rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_order_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_order", + expr=array_check( + "names.rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_side_check() -> Check: + return Check( + field="names.rules[].side", + name="enum", + expr=array_check( + "names.rules", lambda el: check_enum(el["side"], ["left", "right"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _wikidata_check() -> Check: + return Check( + field="wikidata", + name="wikidata_id", + expr=check_pattern( + F.col("wikidata"), + "^Q\\d+\\z", + label="Wikidata identifier (Q followed by digits)", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"wikidata"}), + ) + + +def land_use_checks() -> list[Check]: + """All validation checks for land_use.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _class_required_check(), + _class_enum_check(), + _subtype_required_check(), + _subtype_enum_check(), + _elevation_check(), + _surface_check(), + _names_primary_required_check(), + _names_primary_string_min_length_check(), + _names_primary_stripped_check(), + _names_common_key_check(), + _names_common_value_check(), + _names_rules_value_required_check(), + _names_rules_value_string_min_length_check(), + _names_rules_value_stripped_check(), + _names_rules_variant_required_check(), + _names_rules_variant_enum_check(), + _names_rules_language_check(), + _names_rules_perspectives_mode_required_check(), + _names_rules_perspectives_mode_enum_check(), + _names_rules_perspectives_countries_check(), + _names_rules_perspectives_countries_min_length_check(), + _names_rules_perspectives_countries_unique_check(), + _names_rules_perspectives_countries_check_1(), + _names_rules_between_linear_range_length_check(), + _names_rules_between_linear_range_bounds_check(), + _names_rules_between_linear_range_order_check(), + _names_rules_side_check(), + _wikidata_check(), + ] + + +LAND_USE_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("class", StringType(), True), + StructField("subtype", StringType(), True), + StructField("elevation", IntegerType(), True), + StructField("surface", StringType(), True), + StructField( + "names", + StructType( + [ + StructField("primary", StringType(), True), + StructField( + "common", MapType(StringType(), StringType(), True), True + ), + StructField( + "rules", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("variant", StringType(), True), + StructField("language", StringType(), True), + StructField( + "perspectives", + StructType( + [ + StructField("mode", StringType(), True), + StructField( + "countries", + ArrayType(StringType(), True), + True, + ), + ] + ), + True, + ), + StructField( + "between", ArrayType(DoubleType(), True), True + ), + StructField("side", StringType(), True), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + StructField("level", IntegerType(), True), + StructField("source_tags", MapType(StringType(), StringType(), True), True), + StructField("wikidata", StringType(), True), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = ( + GeometryType.LINE_STRING, + GeometryType.MULTI_POLYGON, + GeometryType.POINT, + GeometryType.POLYGON, +) + +ENTRY_POINT = "overture.schema.base:LandUse" + +PARTITIONS: dict[str, str] = {"theme": "base"} + +MODEL_VALIDATION = ModelValidation( + schema=LAND_USE_SCHEMA, + checks=land_use_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/water.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/water.py new file mode 100644 index 000000000..e94e33fd0 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/base/water.py @@ -0,0 +1,916 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Water validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + BooleanType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, + map_keys_check, + map_values_check, + nested_array_check, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type( + F.col("geometry"), + GeometryType.LINE_STRING, + GeometryType.MULTI_POLYGON, + GeometryType.POINT, + GeometryType.POLYGON, + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["base"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["water"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _class_check() -> Check: + return Check( + field="class", + name="enum", + expr=check_enum( + F.col("class"), + [ + "basin", + "bay", + "blowhole", + "canal", + "cape", + "ditch", + "dock", + "drain", + "fairway", + "fish_pass", + "fishpond", + "geyser", + "hot_spring", + "lagoon", + "lake", + "moat", + "ocean", + "oxbow", + "pond", + "reflecting_pool", + "reservoir", + "river", + "salt_pond", + "sea", + "sewage", + "shoal", + "spring", + "strait", + "stream", + "swimming_pool", + "tidal_channel", + "wastewater", + "water", + "water_storage", + "waterfall", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class"}), + ) + + +def _subtype_check() -> Check: + return Check( + field="subtype", + name="enum", + expr=check_enum( + F.col("subtype"), + [ + "canal", + "human_made", + "lake", + "ocean", + "physical", + "pond", + "reservoir", + "river", + "spring", + "stream", + "wastewater", + "water", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _names_primary_required_check() -> Check: + return Check( + field="names.primary", + name="required", + expr=F.when(F.col("names").isNotNull(), check_required(F.col("names.primary"))), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_string_min_length_check() -> Check: + return Check( + field="names.primary", + name="string_min_length", + expr=check_string_min_length(F.col("names.primary"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_stripped_check() -> Check: + return Check( + field="names.primary", + name="stripped", + expr=check_stripped(F.col("names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_common_key_check() -> Check: + return Check( + field="names.common{key}", + name="language_tag", + expr=map_keys_check( + "names.common", + lambda k: check_pattern( + k, + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_common_value_check() -> Check: + return Check( + field="names.common{value}", + name="stripped", + expr=map_values_check("names.common", lambda v: check_stripped(v)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_required_check() -> Check: + return Check( + field="names.rules[].value", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_string_min_length_check() -> Check: + return Check( + field="names.rules[].value", + name="string_min_length", + expr=array_check( + "names.rules", lambda el: check_string_min_length(el["value"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_stripped_check() -> Check: + return Check( + field="names.rules[].value", + name="stripped", + expr=array_check("names.rules", lambda el: check_stripped(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_required_check() -> Check: + return Check( + field="names.rules[].variant", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["variant"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_enum_check() -> Check: + return Check( + field="names.rules[].variant", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["variant"], ["common", "official", "alternate", "short"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_language_check() -> Check: + return Check( + field="names.rules[].language", + name="language_tag", + expr=array_check( + "names.rules", + lambda el: check_pattern( + el["language"], + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_required_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["mode"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_enum_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["perspectives"]["mode"], ["accepted_by", "disputed_by"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check() -> Check: + return Check( + field="names.rules[].perspectives.countries", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["countries"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_min_length_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_min_length", + name="array_min_length", + expr=array_check( + "names.rules", + lambda el: check_array_min_length(el["perspectives"]["countries"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_unique_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_unique", + name="struct_unique", + expr=array_check( + "names.rules", + lambda el: check_struct_unique(el["perspectives"]["countries"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check_1() -> Check: + return Check( + field="names.rules[].perspectives.countries[]", + name="country_code_alpha2", + expr=nested_array_check( + "names.rules", + lambda el: array_check( + el["perspectives"]["countries"], + lambda inner: check_pattern( + inner, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_length_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_length", + expr=array_check( + "names.rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_bounds", + expr=array_check( + "names.rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_order_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_order", + expr=array_check( + "names.rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_side_check() -> Check: + return Check( + field="names.rules[].side", + name="enum", + expr=array_check( + "names.rules", lambda el: check_enum(el["side"], ["left", "right"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _wikidata_check() -> Check: + return Check( + field="wikidata", + name="wikidata_id", + expr=check_pattern( + F.col("wikidata"), + "^Q\\d+\\z", + label="Wikidata identifier (Q followed by digits)", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"wikidata"}), + ) + + +def water_checks() -> list[Check]: + """All validation checks for water.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _class_check(), + _subtype_check(), + _names_primary_required_check(), + _names_primary_string_min_length_check(), + _names_primary_stripped_check(), + _names_common_key_check(), + _names_common_value_check(), + _names_rules_value_required_check(), + _names_rules_value_string_min_length_check(), + _names_rules_value_stripped_check(), + _names_rules_variant_required_check(), + _names_rules_variant_enum_check(), + _names_rules_language_check(), + _names_rules_perspectives_mode_required_check(), + _names_rules_perspectives_mode_enum_check(), + _names_rules_perspectives_countries_check(), + _names_rules_perspectives_countries_min_length_check(), + _names_rules_perspectives_countries_unique_check(), + _names_rules_perspectives_countries_check_1(), + _names_rules_between_linear_range_length_check(), + _names_rules_between_linear_range_bounds_check(), + _names_rules_between_linear_range_order_check(), + _names_rules_side_check(), + _wikidata_check(), + ] + + +WATER_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("class", StringType(), True), + StructField("subtype", StringType(), True), + StructField("is_intermittent", BooleanType(), True), + StructField("is_salt", BooleanType(), True), + StructField("level", IntegerType(), True), + StructField( + "names", + StructType( + [ + StructField("primary", StringType(), True), + StructField( + "common", MapType(StringType(), StringType(), True), True + ), + StructField( + "rules", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("variant", StringType(), True), + StructField("language", StringType(), True), + StructField( + "perspectives", + StructType( + [ + StructField("mode", StringType(), True), + StructField( + "countries", + ArrayType(StringType(), True), + True, + ), + ] + ), + True, + ), + StructField( + "between", ArrayType(DoubleType(), True), True + ), + StructField("side", StringType(), True), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + StructField("source_tags", MapType(StringType(), StringType(), True), True), + StructField("wikidata", StringType(), True), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = ( + GeometryType.LINE_STRING, + GeometryType.MULTI_POLYGON, + GeometryType.POINT, + GeometryType.POLYGON, +) + +ENTRY_POINT = "overture.schema.base:Water" + +PARTITIONS: dict[str, str] = {"theme": "base"} + +MODEL_VALIDATION = ModelValidation( + schema=WATER_SCHEMA, + checks=water_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/buildings/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/buildings/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/buildings/building.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/buildings/building.py new file mode 100644 index 000000000..40d4f4cc3 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/buildings/building.py @@ -0,0 +1,1150 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Building validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + BooleanType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, + map_keys_check, + map_values_check, + nested_array_check, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type( + F.col("geometry"), GeometryType.MULTI_POLYGON, GeometryType.POLYGON + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["buildings"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["building"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _subtype_check() -> Check: + return Check( + field="subtype", + name="enum", + expr=check_enum( + F.col("subtype"), + [ + "agricultural", + "civic", + "commercial", + "education", + "entertainment", + "industrial", + "medical", + "military", + "outbuilding", + "religious", + "residential", + "service", + "transportation", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _class_check() -> Check: + return Check( + field="class", + name="enum", + expr=check_enum( + F.col("class"), + [ + "agricultural", + "allotment_house", + "apartments", + "barn", + "beach_hut", + "boathouse", + "bridge_structure", + "bungalow", + "bunker", + "cabin", + "carport", + "cathedral", + "chapel", + "church", + "civic", + "college", + "commercial", + "cowshed", + "detached", + "digester", + "dormitory", + "dwelling_house", + "factory", + "farm", + "farm_auxiliary", + "fire_station", + "garage", + "garages", + "ger", + "glasshouse", + "government", + "grandstand", + "greenhouse", + "guardhouse", + "hangar", + "hospital", + "hotel", + "house", + "houseboat", + "hut", + "industrial", + "kindergarten", + "kiosk", + "library", + "manufacture", + "military", + "monastery", + "mosque", + "office", + "outbuilding", + "parking", + "pavilion", + "post_office", + "presbytery", + "public", + "religious", + "residential", + "retail", + "roof", + "school", + "semi", + "semidetached_house", + "service", + "shed", + "shrine", + "silo", + "slurry_tank", + "sports_centre", + "sports_hall", + "stable", + "stadium", + "static_caravan", + "stilt_house", + "storage_tank", + "sty", + "supermarket", + "synagogue", + "temple", + "terrace", + "toilets", + "train_station", + "transformer_tower", + "transportation", + "trullo", + "university", + "warehouse", + "wayside_shrine", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class"}), + ) + + +def _names_primary_required_check() -> Check: + return Check( + field="names.primary", + name="required", + expr=F.when(F.col("names").isNotNull(), check_required(F.col("names.primary"))), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_string_min_length_check() -> Check: + return Check( + field="names.primary", + name="string_min_length", + expr=check_string_min_length(F.col("names.primary"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_stripped_check() -> Check: + return Check( + field="names.primary", + name="stripped", + expr=check_stripped(F.col("names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_common_key_check() -> Check: + return Check( + field="names.common{key}", + name="language_tag", + expr=map_keys_check( + "names.common", + lambda k: check_pattern( + k, + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_common_value_check() -> Check: + return Check( + field="names.common{value}", + name="stripped", + expr=map_values_check("names.common", lambda v: check_stripped(v)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_required_check() -> Check: + return Check( + field="names.rules[].value", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_string_min_length_check() -> Check: + return Check( + field="names.rules[].value", + name="string_min_length", + expr=array_check( + "names.rules", lambda el: check_string_min_length(el["value"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_stripped_check() -> Check: + return Check( + field="names.rules[].value", + name="stripped", + expr=array_check("names.rules", lambda el: check_stripped(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_required_check() -> Check: + return Check( + field="names.rules[].variant", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["variant"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_enum_check() -> Check: + return Check( + field="names.rules[].variant", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["variant"], ["common", "official", "alternate", "short"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_language_check() -> Check: + return Check( + field="names.rules[].language", + name="language_tag", + expr=array_check( + "names.rules", + lambda el: check_pattern( + el["language"], + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_required_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["mode"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_enum_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["perspectives"]["mode"], ["accepted_by", "disputed_by"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check() -> Check: + return Check( + field="names.rules[].perspectives.countries", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["countries"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_min_length_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_min_length", + name="array_min_length", + expr=array_check( + "names.rules", + lambda el: check_array_min_length(el["perspectives"]["countries"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_unique_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_unique", + name="struct_unique", + expr=array_check( + "names.rules", + lambda el: check_struct_unique(el["perspectives"]["countries"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check_1() -> Check: + return Check( + field="names.rules[].perspectives.countries[]", + name="country_code_alpha2", + expr=nested_array_check( + "names.rules", + lambda el: array_check( + el["perspectives"]["countries"], + lambda inner: check_pattern( + inner, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_length_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_length", + expr=array_check( + "names.rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_bounds", + expr=array_check( + "names.rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_order_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_order", + expr=array_check( + "names.rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_side_check() -> Check: + return Check( + field="names.rules[].side", + name="enum", + expr=array_check( + "names.rules", lambda el: check_enum(el["side"], ["left", "right"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _height_check() -> Check: + return Check( + field="height", + name="bounds", + expr=check_bounds(F.col("height"), gt=0.0), + shape=CheckShape.SCALAR, + read_columns=frozenset({"height"}), + ) + + +def _num_floors_check() -> Check: + return Check( + field="num_floors", + name="bounds", + expr=check_bounds(F.col("num_floors"), gt=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"num_floors"}), + ) + + +def _num_floors_underground_check() -> Check: + return Check( + field="num_floors_underground", + name="bounds", + expr=check_bounds(F.col("num_floors_underground"), gt=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"num_floors_underground"}), + ) + + +def _min_floor_check() -> Check: + return Check( + field="min_floor", + name="bounds", + expr=check_bounds(F.col("min_floor"), gt=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"min_floor"}), + ) + + +def _facade_color_check() -> Check: + return Check( + field="facade_color", + name="hex_color", + expr=check_pattern( + F.col("facade_color"), + "^#[0-9A-Fa-f]{3}([0-9A-Fa-f]{3})?\\z", + label="Hexadecimal color code in format #RGB or #RRGGBB", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"facade_color"}), + ) + + +def _facade_material_check() -> Check: + return Check( + field="facade_material", + name="enum", + expr=check_enum( + F.col("facade_material"), + [ + "brick", + "cement_block", + "clay", + "concrete", + "glass", + "metal", + "plaster", + "plastic", + "stone", + "timber_framing", + "wood", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"facade_material"}), + ) + + +def _roof_material_check() -> Check: + return Check( + field="roof_material", + name="enum", + expr=check_enum( + F.col("roof_material"), + [ + "concrete", + "copper", + "eternit", + "glass", + "grass", + "gravel", + "metal", + "plastic", + "roof_tiles", + "slate", + "solar_panels", + "tar_paper", + "thatch", + "wood", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"roof_material"}), + ) + + +def _roof_shape_check() -> Check: + return Check( + field="roof_shape", + name="enum", + expr=check_enum( + F.col("roof_shape"), + [ + "dome", + "flat", + "gabled", + "gambrel", + "half_hipped", + "hipped", + "mansard", + "onion", + "pyramidal", + "round", + "saltbox", + "sawtooth", + "skillion", + "spherical", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"roof_shape"}), + ) + + +def _roof_direction_bounds_check() -> Check: + return Check( + field="roof_direction_0", + name="bounds", + expr=check_bounds(F.col("roof_direction"), ge=0.0), + shape=CheckShape.SCALAR, + read_columns=frozenset({"roof_direction"}), + ) + + +def _roof_direction_bounds_check_1() -> Check: + return Check( + field="roof_direction_1", + name="bounds", + expr=check_bounds(F.col("roof_direction"), lt=360.0), + shape=CheckShape.SCALAR, + read_columns=frozenset({"roof_direction"}), + ) + + +def _roof_orientation_check() -> Check: + return Check( + field="roof_orientation", + name="enum", + expr=check_enum(F.col("roof_orientation"), ["across", "along"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"roof_orientation"}), + ) + + +def _roof_color_check() -> Check: + return Check( + field="roof_color", + name="hex_color", + expr=check_pattern( + F.col("roof_color"), + "^#[0-9A-Fa-f]{3}([0-9A-Fa-f]{3})?\\z", + label="Hexadecimal color code in format #RGB or #RRGGBB", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"roof_color"}), + ) + + +def building_checks() -> list[Check]: + """All validation checks for building.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _subtype_check(), + _class_check(), + _names_primary_required_check(), + _names_primary_string_min_length_check(), + _names_primary_stripped_check(), + _names_common_key_check(), + _names_common_value_check(), + _names_rules_value_required_check(), + _names_rules_value_string_min_length_check(), + _names_rules_value_stripped_check(), + _names_rules_variant_required_check(), + _names_rules_variant_enum_check(), + _names_rules_language_check(), + _names_rules_perspectives_mode_required_check(), + _names_rules_perspectives_mode_enum_check(), + _names_rules_perspectives_countries_check(), + _names_rules_perspectives_countries_min_length_check(), + _names_rules_perspectives_countries_unique_check(), + _names_rules_perspectives_countries_check_1(), + _names_rules_between_linear_range_length_check(), + _names_rules_between_linear_range_bounds_check(), + _names_rules_between_linear_range_order_check(), + _names_rules_side_check(), + _height_check(), + _num_floors_check(), + _num_floors_underground_check(), + _min_floor_check(), + _facade_color_check(), + _facade_material_check(), + _roof_material_check(), + _roof_shape_check(), + _roof_direction_bounds_check(), + _roof_direction_bounds_check_1(), + _roof_orientation_check(), + _roof_color_check(), + ] + + +BUILDING_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("subtype", StringType(), True), + StructField("class", StringType(), True), + StructField("has_parts", BooleanType(), True), + StructField( + "names", + StructType( + [ + StructField("primary", StringType(), True), + StructField( + "common", MapType(StringType(), StringType(), True), True + ), + StructField( + "rules", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("variant", StringType(), True), + StructField("language", StringType(), True), + StructField( + "perspectives", + StructType( + [ + StructField("mode", StringType(), True), + StructField( + "countries", + ArrayType(StringType(), True), + True, + ), + ] + ), + True, + ), + StructField( + "between", ArrayType(DoubleType(), True), True + ), + StructField("side", StringType(), True), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + StructField("level", IntegerType(), True), + StructField("height", DoubleType(), True), + StructField("is_underground", BooleanType(), True), + StructField("num_floors", IntegerType(), True), + StructField("num_floors_underground", IntegerType(), True), + StructField("min_height", DoubleType(), True), + StructField("min_floor", IntegerType(), True), + StructField("facade_color", StringType(), True), + StructField("facade_material", StringType(), True), + StructField("roof_material", StringType(), True), + StructField("roof_shape", StringType(), True), + StructField("roof_direction", DoubleType(), True), + StructField("roof_orientation", StringType(), True), + StructField("roof_color", StringType(), True), + StructField("roof_height", DoubleType(), True), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = ( + GeometryType.MULTI_POLYGON, + GeometryType.POLYGON, +) + +ENTRY_POINT = "overture.schema.buildings:Building" + +PARTITIONS: dict[str, str] = {"theme": "buildings"} + +MODEL_VALIDATION = ModelValidation( + schema=BUILDING_SCHEMA, + checks=building_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/buildings/building_part.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/buildings/building_part.py new file mode 100644 index 000000000..6e19a7bb3 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/buildings/building_part.py @@ -0,0 +1,1055 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Building Part validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + BooleanType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, + map_keys_check, + map_values_check, + nested_array_check, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type( + F.col("geometry"), GeometryType.MULTI_POLYGON, GeometryType.POLYGON + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["buildings"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["building_part"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _building_id_required_check() -> Check: + return Check( + field="building_id", + name="required", + expr=check_required(F.col("building_id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"building_id"}), + ) + + +def _building_id_string_min_length_check() -> Check: + return Check( + field="building_id", + name="string_min_length", + expr=check_string_min_length(F.col("building_id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"building_id"}), + ) + + +def _building_id_no_whitespace_check() -> Check: + return Check( + field="building_id", + name="no_whitespace", + expr=check_pattern( + F.col("building_id"), + "^\\S+\\z", + label="String without whitespace characters", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"building_id"}), + ) + + +def _names_primary_required_check() -> Check: + return Check( + field="names.primary", + name="required", + expr=F.when(F.col("names").isNotNull(), check_required(F.col("names.primary"))), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_string_min_length_check() -> Check: + return Check( + field="names.primary", + name="string_min_length", + expr=check_string_min_length(F.col("names.primary"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_stripped_check() -> Check: + return Check( + field="names.primary", + name="stripped", + expr=check_stripped(F.col("names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_common_key_check() -> Check: + return Check( + field="names.common{key}", + name="language_tag", + expr=map_keys_check( + "names.common", + lambda k: check_pattern( + k, + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_common_value_check() -> Check: + return Check( + field="names.common{value}", + name="stripped", + expr=map_values_check("names.common", lambda v: check_stripped(v)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_required_check() -> Check: + return Check( + field="names.rules[].value", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_string_min_length_check() -> Check: + return Check( + field="names.rules[].value", + name="string_min_length", + expr=array_check( + "names.rules", lambda el: check_string_min_length(el["value"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_stripped_check() -> Check: + return Check( + field="names.rules[].value", + name="stripped", + expr=array_check("names.rules", lambda el: check_stripped(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_required_check() -> Check: + return Check( + field="names.rules[].variant", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["variant"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_enum_check() -> Check: + return Check( + field="names.rules[].variant", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["variant"], ["common", "official", "alternate", "short"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_language_check() -> Check: + return Check( + field="names.rules[].language", + name="language_tag", + expr=array_check( + "names.rules", + lambda el: check_pattern( + el["language"], + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_required_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["mode"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_enum_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["perspectives"]["mode"], ["accepted_by", "disputed_by"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check() -> Check: + return Check( + field="names.rules[].perspectives.countries", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["countries"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_min_length_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_min_length", + name="array_min_length", + expr=array_check( + "names.rules", + lambda el: check_array_min_length(el["perspectives"]["countries"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_unique_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_unique", + name="struct_unique", + expr=array_check( + "names.rules", + lambda el: check_struct_unique(el["perspectives"]["countries"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check_1() -> Check: + return Check( + field="names.rules[].perspectives.countries[]", + name="country_code_alpha2", + expr=nested_array_check( + "names.rules", + lambda el: array_check( + el["perspectives"]["countries"], + lambda inner: check_pattern( + inner, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_length_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_length", + expr=array_check( + "names.rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_bounds", + expr=array_check( + "names.rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_order_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_order", + expr=array_check( + "names.rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_side_check() -> Check: + return Check( + field="names.rules[].side", + name="enum", + expr=array_check( + "names.rules", lambda el: check_enum(el["side"], ["left", "right"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _height_check() -> Check: + return Check( + field="height", + name="bounds", + expr=check_bounds(F.col("height"), gt=0.0), + shape=CheckShape.SCALAR, + read_columns=frozenset({"height"}), + ) + + +def _num_floors_check() -> Check: + return Check( + field="num_floors", + name="bounds", + expr=check_bounds(F.col("num_floors"), gt=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"num_floors"}), + ) + + +def _num_floors_underground_check() -> Check: + return Check( + field="num_floors_underground", + name="bounds", + expr=check_bounds(F.col("num_floors_underground"), gt=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"num_floors_underground"}), + ) + + +def _min_floor_check() -> Check: + return Check( + field="min_floor", + name="bounds", + expr=check_bounds(F.col("min_floor"), gt=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"min_floor"}), + ) + + +def _facade_color_check() -> Check: + return Check( + field="facade_color", + name="hex_color", + expr=check_pattern( + F.col("facade_color"), + "^#[0-9A-Fa-f]{3}([0-9A-Fa-f]{3})?\\z", + label="Hexadecimal color code in format #RGB or #RRGGBB", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"facade_color"}), + ) + + +def _facade_material_check() -> Check: + return Check( + field="facade_material", + name="enum", + expr=check_enum( + F.col("facade_material"), + [ + "brick", + "cement_block", + "clay", + "concrete", + "glass", + "metal", + "plaster", + "plastic", + "stone", + "timber_framing", + "wood", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"facade_material"}), + ) + + +def _roof_material_check() -> Check: + return Check( + field="roof_material", + name="enum", + expr=check_enum( + F.col("roof_material"), + [ + "concrete", + "copper", + "eternit", + "glass", + "grass", + "gravel", + "metal", + "plastic", + "roof_tiles", + "slate", + "solar_panels", + "tar_paper", + "thatch", + "wood", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"roof_material"}), + ) + + +def _roof_shape_check() -> Check: + return Check( + field="roof_shape", + name="enum", + expr=check_enum( + F.col("roof_shape"), + [ + "dome", + "flat", + "gabled", + "gambrel", + "half_hipped", + "hipped", + "mansard", + "onion", + "pyramidal", + "round", + "saltbox", + "sawtooth", + "skillion", + "spherical", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"roof_shape"}), + ) + + +def _roof_direction_bounds_check() -> Check: + return Check( + field="roof_direction_0", + name="bounds", + expr=check_bounds(F.col("roof_direction"), ge=0.0), + shape=CheckShape.SCALAR, + read_columns=frozenset({"roof_direction"}), + ) + + +def _roof_direction_bounds_check_1() -> Check: + return Check( + field="roof_direction_1", + name="bounds", + expr=check_bounds(F.col("roof_direction"), lt=360.0), + shape=CheckShape.SCALAR, + read_columns=frozenset({"roof_direction"}), + ) + + +def _roof_orientation_check() -> Check: + return Check( + field="roof_orientation", + name="enum", + expr=check_enum(F.col("roof_orientation"), ["across", "along"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"roof_orientation"}), + ) + + +def _roof_color_check() -> Check: + return Check( + field="roof_color", + name="hex_color", + expr=check_pattern( + F.col("roof_color"), + "^#[0-9A-Fa-f]{3}([0-9A-Fa-f]{3})?\\z", + label="Hexadecimal color code in format #RGB or #RRGGBB", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"roof_color"}), + ) + + +def building_part_checks() -> list[Check]: + """All validation checks for building_part.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _building_id_required_check(), + _building_id_string_min_length_check(), + _building_id_no_whitespace_check(), + _names_primary_required_check(), + _names_primary_string_min_length_check(), + _names_primary_stripped_check(), + _names_common_key_check(), + _names_common_value_check(), + _names_rules_value_required_check(), + _names_rules_value_string_min_length_check(), + _names_rules_value_stripped_check(), + _names_rules_variant_required_check(), + _names_rules_variant_enum_check(), + _names_rules_language_check(), + _names_rules_perspectives_mode_required_check(), + _names_rules_perspectives_mode_enum_check(), + _names_rules_perspectives_countries_check(), + _names_rules_perspectives_countries_min_length_check(), + _names_rules_perspectives_countries_unique_check(), + _names_rules_perspectives_countries_check_1(), + _names_rules_between_linear_range_length_check(), + _names_rules_between_linear_range_bounds_check(), + _names_rules_between_linear_range_order_check(), + _names_rules_side_check(), + _height_check(), + _num_floors_check(), + _num_floors_underground_check(), + _min_floor_check(), + _facade_color_check(), + _facade_material_check(), + _roof_material_check(), + _roof_shape_check(), + _roof_direction_bounds_check(), + _roof_direction_bounds_check_1(), + _roof_orientation_check(), + _roof_color_check(), + ] + + +BUILDING_PART_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("building_id", StringType(), True), + StructField( + "names", + StructType( + [ + StructField("primary", StringType(), True), + StructField( + "common", MapType(StringType(), StringType(), True), True + ), + StructField( + "rules", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("variant", StringType(), True), + StructField("language", StringType(), True), + StructField( + "perspectives", + StructType( + [ + StructField("mode", StringType(), True), + StructField( + "countries", + ArrayType(StringType(), True), + True, + ), + ] + ), + True, + ), + StructField( + "between", ArrayType(DoubleType(), True), True + ), + StructField("side", StringType(), True), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + StructField("level", IntegerType(), True), + StructField("height", DoubleType(), True), + StructField("is_underground", BooleanType(), True), + StructField("num_floors", IntegerType(), True), + StructField("num_floors_underground", IntegerType(), True), + StructField("min_height", DoubleType(), True), + StructField("min_floor", IntegerType(), True), + StructField("facade_color", StringType(), True), + StructField("facade_material", StringType(), True), + StructField("roof_material", StringType(), True), + StructField("roof_shape", StringType(), True), + StructField("roof_direction", DoubleType(), True), + StructField("roof_orientation", StringType(), True), + StructField("roof_color", StringType(), True), + StructField("roof_height", DoubleType(), True), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = ( + GeometryType.MULTI_POLYGON, + GeometryType.POLYGON, +) + +ENTRY_POINT = "overture.schema.buildings:BuildingPart" + +PARTITIONS: dict[str, str] = {"theme": "buildings"} + +MODEL_VALIDATION = ModelValidation( + schema=BUILDING_PART_SCHEMA, + checks=building_part_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/divisions/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/divisions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/divisions/division.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/divisions/division.py new file mode 100644 index 000000000..487c6150c --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/divisions/division.py @@ -0,0 +1,1704 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Division validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, + map_keys_check, + map_values_check, + nested_array_check, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_forbid_if, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_require_if, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _cartography_prominence_bounds_check() -> Check: + return Check( + field="cartography.prominence_0", + name="bounds", + expr=check_bounds(F.col("cartography.prominence"), ge=1, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_prominence_bounds_check_1() -> Check: + return Check( + field="cartography.prominence_1", + name="bounds", + expr=check_bounds(F.col("cartography.prominence"), le=100, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_min_zoom_bounds_check() -> Check: + return Check( + field="cartography.min_zoom_0", + name="bounds", + expr=check_bounds(F.col("cartography.min_zoom"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_min_zoom_bounds_check_1() -> Check: + return Check( + field="cartography.min_zoom_1", + name="bounds", + expr=check_bounds(F.col("cartography.min_zoom"), le=23, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_max_zoom_bounds_check() -> Check: + return Check( + field="cartography.max_zoom_0", + name="bounds", + expr=check_bounds(F.col("cartography.max_zoom"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _cartography_max_zoom_bounds_check_1() -> Check: + return Check( + field="cartography.max_zoom_1", + name="bounds", + expr=check_bounds(F.col("cartography.max_zoom"), le=23, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"cartography"}), + ) + + +def _names_check() -> Check: + return Check( + field="names", + name="required", + expr=check_required(F.col("names")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_required_check() -> Check: + return Check( + field="names.primary", + name="required", + expr=check_required(F.col("names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_string_min_length_check() -> Check: + return Check( + field="names.primary", + name="string_min_length", + expr=check_string_min_length(F.col("names.primary"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_stripped_check() -> Check: + return Check( + field="names.primary", + name="stripped", + expr=check_stripped(F.col("names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_common_key_check() -> Check: + return Check( + field="names.common{key}", + name="language_tag", + expr=map_keys_check( + "names.common", + lambda k: check_pattern( + k, + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_common_value_check() -> Check: + return Check( + field="names.common{value}", + name="stripped", + expr=map_values_check("names.common", lambda v: check_stripped(v)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_required_check() -> Check: + return Check( + field="names.rules[].value", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_string_min_length_check() -> Check: + return Check( + field="names.rules[].value", + name="string_min_length", + expr=array_check( + "names.rules", lambda el: check_string_min_length(el["value"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_stripped_check() -> Check: + return Check( + field="names.rules[].value", + name="stripped", + expr=array_check("names.rules", lambda el: check_stripped(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_required_check() -> Check: + return Check( + field="names.rules[].variant", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["variant"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_enum_check() -> Check: + return Check( + field="names.rules[].variant", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["variant"], ["common", "official", "alternate", "short"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_language_check() -> Check: + return Check( + field="names.rules[].language", + name="language_tag", + expr=array_check( + "names.rules", + lambda el: check_pattern( + el["language"], + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_required_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["mode"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_enum_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["perspectives"]["mode"], ["accepted_by", "disputed_by"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check() -> Check: + return Check( + field="names.rules[].perspectives.countries", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["countries"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_min_length_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_min_length", + name="array_min_length", + expr=array_check( + "names.rules", + lambda el: check_array_min_length(el["perspectives"]["countries"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_unique_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_unique", + name="struct_unique", + expr=array_check( + "names.rules", + lambda el: check_struct_unique(el["perspectives"]["countries"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check_1() -> Check: + return Check( + field="names.rules[].perspectives.countries[]", + name="country_code_alpha2", + expr=nested_array_check( + "names.rules", + lambda el: array_check( + el["perspectives"]["countries"], + lambda inner: check_pattern( + inner, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_length_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_length", + expr=array_check( + "names.rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_bounds", + expr=array_check( + "names.rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_order_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_order", + expr=array_check( + "names.rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_side_check() -> Check: + return Check( + field="names.rules[].side", + name="enum", + expr=array_check( + "names.rules", lambda el: check_enum(el["side"], ["left", "right"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type(F.col("geometry"), GeometryType.POINT), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["divisions"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["division"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _subtype_required_check() -> Check: + return Check( + field="subtype", + name="required", + expr=check_required(F.col("subtype")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _subtype_enum_check() -> Check: + return Check( + field="subtype", + name="enum", + expr=check_enum( + F.col("subtype"), + [ + "country", + "dependency", + "macroregion", + "region", + "macrocounty", + "county", + "localadmin", + "locality", + "borough", + "macrohood", + "neighborhood", + "microhood", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _country_required_check() -> Check: + return Check( + field="country", + name="required", + expr=check_required(F.col("country")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"country"}), + ) + + +def _country_country_code_alpha2_check() -> Check: + return Check( + field="country", + name="country_code_alpha2", + expr=check_pattern( + F.col("country"), "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"country"}), + ) + + +def _hierarchies_check() -> Check: + return Check( + field="hierarchies", + name="required", + expr=check_required(F.col("hierarchies")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"hierarchies"}), + ) + + +def _hierarchies_min_length_check() -> Check: + return Check( + field="hierarchies_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("hierarchies"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"hierarchies"}), + ) + + +def _hierarchies_unique_check() -> Check: + return Check( + field="hierarchies_unique", + name="struct_unique", + expr=check_struct_unique(F.col("hierarchies")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"hierarchies"}), + ) + + +def _hierarchies_min_length_check_1() -> Check: + return Check( + field="hierarchies[]_min_length", + name="array_min_length", + expr=array_check("hierarchies", lambda el: check_array_min_length(el, 1)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"hierarchies"}), + ) + + +def _hierarchies_unique_check_1() -> Check: + return Check( + field="hierarchies[]_unique", + name="struct_unique", + expr=array_check("hierarchies", lambda el: check_struct_unique(el)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"hierarchies"}), + ) + + +def _hierarchies_division_id_required_check() -> Check: + return Check( + field="hierarchies[][].division_id", + name="required", + expr=nested_array_check( + "hierarchies", + lambda el: array_check( + el, lambda inner: check_required(inner["division_id"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"hierarchies"}), + ) + + +def _hierarchies_division_id_string_min_length_check() -> Check: + return Check( + field="hierarchies[][].division_id", + name="string_min_length", + expr=nested_array_check( + "hierarchies", + lambda el: array_check( + el, lambda inner: check_string_min_length(inner["division_id"], 1) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"hierarchies"}), + ) + + +def _hierarchies_division_id_no_whitespace_check() -> Check: + return Check( + field="hierarchies[][].division_id", + name="no_whitespace", + expr=nested_array_check( + "hierarchies", + lambda el: array_check( + el, + lambda inner: check_pattern( + inner["division_id"], + "^\\S+\\z", + label="String without whitespace characters", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"hierarchies"}), + ) + + +def _hierarchies_subtype_required_check() -> Check: + return Check( + field="hierarchies[][].subtype", + name="required", + expr=nested_array_check( + "hierarchies", + lambda el: array_check(el, lambda inner: check_required(inner["subtype"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"hierarchies"}), + ) + + +def _hierarchies_subtype_enum_check() -> Check: + return Check( + field="hierarchies[][].subtype", + name="enum", + expr=nested_array_check( + "hierarchies", + lambda el: array_check( + el, + lambda inner: check_enum( + inner["subtype"], + [ + "country", + "dependency", + "macroregion", + "region", + "macrocounty", + "county", + "localadmin", + "locality", + "borough", + "macrohood", + "neighborhood", + "microhood", + ], + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"hierarchies"}), + ) + + +def _hierarchies_name_required_check() -> Check: + return Check( + field="hierarchies[][].name", + name="required", + expr=nested_array_check( + "hierarchies", + lambda el: array_check(el, lambda inner: check_required(inner["name"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"hierarchies"}), + ) + + +def _hierarchies_name_string_min_length_check() -> Check: + return Check( + field="hierarchies[][].name", + name="string_min_length", + expr=nested_array_check( + "hierarchies", + lambda el: array_check( + el, lambda inner: check_string_min_length(inner["name"], 1) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"hierarchies"}), + ) + + +def _hierarchies_name_stripped_check() -> Check: + return Check( + field="hierarchies[][].name", + name="stripped", + expr=nested_array_check( + "hierarchies", + lambda el: array_check(el, lambda inner: check_stripped(inner["name"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"hierarchies"}), + ) + + +def _parent_division_id_string_min_length_check() -> Check: + return Check( + field="parent_division_id", + name="string_min_length", + expr=check_string_min_length(F.col("parent_division_id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"parent_division_id"}), + ) + + +def _parent_division_id_no_whitespace_check() -> Check: + return Check( + field="parent_division_id", + name="no_whitespace", + expr=check_pattern( + F.col("parent_division_id"), + "^\\S+\\z", + label="String without whitespace characters", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"parent_division_id"}), + ) + + +def _admin_level_bounds_check() -> Check: + return Check( + field="admin_level_0", + name="bounds", + expr=check_bounds(F.col("admin_level"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level"}), + ) + + +def _admin_level_bounds_check_1() -> Check: + return Check( + field="admin_level_1", + name="bounds", + expr=check_bounds(F.col("admin_level"), le=16, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level"}), + ) + + +def _class_check() -> Check: + return Check( + field="class", + name="enum", + expr=check_enum( + F.col("class"), ["megacity", "city", "town", "village", "hamlet"] + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class"}), + ) + + +def _local_type_key_check() -> Check: + return Check( + field="local_type{key}", + name="language_tag", + expr=map_keys_check( + "local_type", + lambda k: check_pattern( + k, + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"local_type"}), + ) + + +def _local_type_value_check() -> Check: + return Check( + field="local_type{value}", + name="stripped", + expr=map_values_check("local_type", lambda v: check_stripped(v)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"local_type"}), + ) + + +def _region_check() -> Check: + return Check( + field="region", + name="region_code", + expr=check_pattern( + F.col("region"), + "^[A-Z]{2}-[A-Z0-9]{1,3}\\z", + label="ISO 3166-2 subdivision code", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"region"}), + ) + + +def _perspectives_mode_required_check() -> Check: + return Check( + field="perspectives.mode", + name="required", + expr=F.when( + F.col("perspectives").isNotNull(), + check_required(F.col("perspectives.mode")), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"perspectives"}), + ) + + +def _perspectives_mode_enum_check() -> Check: + return Check( + field="perspectives.mode", + name="enum", + expr=check_enum(F.col("perspectives.mode"), ["accepted_by", "disputed_by"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"perspectives"}), + ) + + +def _perspectives_countries_check() -> Check: + return Check( + field="perspectives.countries", + name="required", + expr=F.when( + F.col("perspectives").isNotNull(), + check_required(F.col("perspectives.countries")), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"perspectives"}), + ) + + +def _perspectives_countries_min_length_check() -> Check: + return Check( + field="perspectives.countries_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("perspectives.countries"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"perspectives"}), + ) + + +def _perspectives_countries_unique_check() -> Check: + return Check( + field="perspectives.countries_unique", + name="struct_unique", + expr=check_struct_unique(F.col("perspectives.countries")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"perspectives"}), + ) + + +def _perspectives_countries_check_1() -> Check: + return Check( + field="perspectives.countries[]", + name="country_code_alpha2", + expr=array_check( + "perspectives.countries", + lambda el: check_pattern( + el, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"perspectives"}), + ) + + +def _norms_driving_side_check() -> Check: + return Check( + field="norms.driving_side", + name="enum", + expr=check_enum(F.col("norms.driving_side"), ["left", "right"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"norms"}), + ) + + +def _population_check() -> Check: + return Check( + field="population", + name="bounds", + expr=check_bounds(F.col("population"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"population"}), + ) + + +def _capital_division_ids_min_length_check() -> Check: + return Check( + field="capital_division_ids_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("capital_division_ids"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"capital_division_ids"}), + ) + + +def _capital_division_ids_unique_check() -> Check: + return Check( + field="capital_division_ids_unique", + name="struct_unique", + expr=check_struct_unique(F.col("capital_division_ids")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"capital_division_ids"}), + ) + + +def _capital_division_ids_string_min_length_check() -> Check: + return Check( + field="capital_division_ids[]", + name="string_min_length", + expr=array_check( + "capital_division_ids", lambda el: check_string_min_length(el, 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"capital_division_ids"}), + ) + + +def _capital_division_ids_no_whitespace_check() -> Check: + return Check( + field="capital_division_ids[]", + name="no_whitespace", + expr=array_check( + "capital_division_ids", + lambda el: check_pattern( + el, "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"capital_division_ids"}), + ) + + +def _capital_of_divisions_min_length_check() -> Check: + return Check( + field="capital_of_divisions_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("capital_of_divisions"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"capital_of_divisions"}), + ) + + +def _capital_of_divisions_unique_check() -> Check: + return Check( + field="capital_of_divisions_unique", + name="struct_unique", + expr=check_struct_unique(F.col("capital_of_divisions")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"capital_of_divisions"}), + ) + + +def _capital_of_divisions_division_id_required_check() -> Check: + return Check( + field="capital_of_divisions[].division_id", + name="required", + expr=array_check( + "capital_of_divisions", lambda el: check_required(el["division_id"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"capital_of_divisions"}), + ) + + +def _capital_of_divisions_division_id_string_min_length_check() -> Check: + return Check( + field="capital_of_divisions[].division_id", + name="string_min_length", + expr=array_check( + "capital_of_divisions", + lambda el: check_string_min_length(el["division_id"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"capital_of_divisions"}), + ) + + +def _capital_of_divisions_division_id_no_whitespace_check() -> Check: + return Check( + field="capital_of_divisions[].division_id", + name="no_whitespace", + expr=array_check( + "capital_of_divisions", + lambda el: check_pattern( + el["division_id"], + "^\\S+\\z", + label="String without whitespace characters", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"capital_of_divisions"}), + ) + + +def _capital_of_divisions_subtype_required_check() -> Check: + return Check( + field="capital_of_divisions[].subtype", + name="required", + expr=array_check( + "capital_of_divisions", lambda el: check_required(el["subtype"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"capital_of_divisions"}), + ) + + +def _capital_of_divisions_subtype_enum_check() -> Check: + return Check( + field="capital_of_divisions[].subtype", + name="enum", + expr=array_check( + "capital_of_divisions", + lambda el: check_enum( + el["subtype"], + [ + "country", + "dependency", + "macroregion", + "region", + "macrocounty", + "county", + "localadmin", + "locality", + "borough", + "macrohood", + "neighborhood", + "microhood", + ], + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"capital_of_divisions"}), + ) + + +def _wikidata_check() -> Check: + return Check( + field="wikidata", + name="wikidata_id", + expr=check_pattern( + F.col("wikidata"), + "^Q\\d+\\z", + label="Wikidata identifier (Q followed by digits)", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"wikidata"}), + ) + + +def _check_require_if_0_check() -> Check: + return Check( + field="admin_level_required_0", + name="require_if", + expr=check_require_if( + F.col("admin_level"), F.col("subtype") == "county", "subtype = 'county'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_1_check() -> Check: + return Check( + field="admin_level_required_1", + name="require_if", + expr=check_require_if( + F.col("admin_level"), + F.col("subtype") == "macrocounty", + "subtype = 'macrocounty'", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_2_check() -> Check: + return Check( + field="admin_level_required_2", + name="require_if", + expr=check_require_if( + F.col("admin_level"), F.col("subtype") == "region", "subtype = 'region'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_3_check() -> Check: + return Check( + field="admin_level_required_3", + name="require_if", + expr=check_require_if( + F.col("admin_level"), + F.col("subtype") == "macroregion", + "subtype = 'macroregion'", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_4_check() -> Check: + return Check( + field="admin_level_required_4", + name="require_if", + expr=check_require_if( + F.col("admin_level"), + F.col("subtype") == "dependency", + "subtype = 'dependency'", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_5_check() -> Check: + return Check( + field="admin_level_required_5", + name="require_if", + expr=check_require_if( + F.col("admin_level"), F.col("subtype") == "country", "subtype = 'country'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_6_check() -> Check: + return Check( + field="parent_division_id_required", + name="require_if", + expr=check_require_if( + F.col("parent_division_id"), + F.col("subtype") != "country", + "subtype != 'country'", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"parent_division_id", "subtype"}), + ) + + +def _check_forbid_if_7_check() -> Check: + return Check( + field="parent_division_id_forbidden", + name="forbid_if", + expr=check_forbid_if( + F.col("parent_division_id"), + F.col("subtype") == "country", + "subtype = 'country'", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"parent_division_id", "subtype"}), + ) + + +def division_checks() -> list[Check]: + """All validation checks for division.""" + return [ + _cartography_prominence_bounds_check(), + _cartography_prominence_bounds_check_1(), + _cartography_min_zoom_bounds_check(), + _cartography_min_zoom_bounds_check_1(), + _cartography_max_zoom_bounds_check(), + _cartography_max_zoom_bounds_check_1(), + _names_check(), + _names_primary_required_check(), + _names_primary_string_min_length_check(), + _names_primary_stripped_check(), + _names_common_key_check(), + _names_common_value_check(), + _names_rules_value_required_check(), + _names_rules_value_string_min_length_check(), + _names_rules_value_stripped_check(), + _names_rules_variant_required_check(), + _names_rules_variant_enum_check(), + _names_rules_language_check(), + _names_rules_perspectives_mode_required_check(), + _names_rules_perspectives_mode_enum_check(), + _names_rules_perspectives_countries_check(), + _names_rules_perspectives_countries_min_length_check(), + _names_rules_perspectives_countries_unique_check(), + _names_rules_perspectives_countries_check_1(), + _names_rules_between_linear_range_length_check(), + _names_rules_between_linear_range_bounds_check(), + _names_rules_between_linear_range_order_check(), + _names_rules_side_check(), + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _subtype_required_check(), + _subtype_enum_check(), + _country_required_check(), + _country_country_code_alpha2_check(), + _hierarchies_check(), + _hierarchies_min_length_check(), + _hierarchies_unique_check(), + _hierarchies_min_length_check_1(), + _hierarchies_unique_check_1(), + _hierarchies_division_id_required_check(), + _hierarchies_division_id_string_min_length_check(), + _hierarchies_division_id_no_whitespace_check(), + _hierarchies_subtype_required_check(), + _hierarchies_subtype_enum_check(), + _hierarchies_name_required_check(), + _hierarchies_name_string_min_length_check(), + _hierarchies_name_stripped_check(), + _parent_division_id_string_min_length_check(), + _parent_division_id_no_whitespace_check(), + _admin_level_bounds_check(), + _admin_level_bounds_check_1(), + _class_check(), + _local_type_key_check(), + _local_type_value_check(), + _region_check(), + _perspectives_mode_required_check(), + _perspectives_mode_enum_check(), + _perspectives_countries_check(), + _perspectives_countries_min_length_check(), + _perspectives_countries_unique_check(), + _perspectives_countries_check_1(), + _norms_driving_side_check(), + _population_check(), + _capital_division_ids_min_length_check(), + _capital_division_ids_unique_check(), + _capital_division_ids_string_min_length_check(), + _capital_division_ids_no_whitespace_check(), + _capital_of_divisions_min_length_check(), + _capital_of_divisions_unique_check(), + _capital_of_divisions_division_id_required_check(), + _capital_of_divisions_division_id_string_min_length_check(), + _capital_of_divisions_division_id_no_whitespace_check(), + _capital_of_divisions_subtype_required_check(), + _capital_of_divisions_subtype_enum_check(), + _wikidata_check(), + _check_require_if_0_check(), + _check_require_if_1_check(), + _check_require_if_2_check(), + _check_require_if_3_check(), + _check_require_if_4_check(), + _check_require_if_5_check(), + _check_require_if_6_check(), + _check_forbid_if_7_check(), + ] + + +DIVISION_SCHEMA = StructType( + [ + StructField( + "cartography", + StructType( + [ + StructField("prominence", IntegerType(), True), + StructField("min_zoom", IntegerType(), True), + StructField("max_zoom", IntegerType(), True), + StructField("sort_key", IntegerType(), True), + ] + ), + True, + ), + StructField( + "names", + StructType( + [ + StructField("primary", StringType(), True), + StructField( + "common", MapType(StringType(), StringType(), True), True + ), + StructField( + "rules", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("variant", StringType(), True), + StructField("language", StringType(), True), + StructField( + "perspectives", + StructType( + [ + StructField("mode", StringType(), True), + StructField( + "countries", + ArrayType(StringType(), True), + True, + ), + ] + ), + True, + ), + StructField( + "between", ArrayType(DoubleType(), True), True + ), + StructField("side", StringType(), True), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("subtype", StringType(), True), + StructField("country", StringType(), True), + StructField( + "hierarchies", + ArrayType( + ArrayType( + StructType( + [ + StructField("division_id", StringType(), True), + StructField("subtype", StringType(), True), + StructField("name", StringType(), True), + ] + ), + True, + ), + True, + ), + True, + ), + StructField("parent_division_id", StringType(), True), + StructField("admin_level", IntegerType(), True), + StructField("class", StringType(), True), + StructField("local_type", MapType(StringType(), StringType(), True), True), + StructField("region", StringType(), True), + StructField( + "perspectives", + StructType( + [ + StructField("mode", StringType(), True), + StructField("countries", ArrayType(StringType(), True), True), + ] + ), + True, + ), + StructField( + "norms", StructType([StructField("driving_side", StringType(), True)]), True + ), + StructField("population", IntegerType(), True), + StructField("capital_division_ids", ArrayType(StringType(), True), True), + StructField( + "capital_of_divisions", + ArrayType( + StructType( + [ + StructField("division_id", StringType(), True), + StructField("subtype", StringType(), True), + ] + ), + True, + ), + True, + ), + StructField("wikidata", StringType(), True), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = (GeometryType.POINT,) + +ENTRY_POINT = "overture.schema.divisions:Division" + +PARTITIONS: dict[str, str] = {"theme": "divisions"} + +MODEL_VALIDATION = ModelValidation( + schema=DIVISION_SCHEMA, + checks=division_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/divisions/division_area.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/divisions/division_area.py new file mode 100644 index 000000000..9b83b124b --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/divisions/division_area.py @@ -0,0 +1,1088 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Division Area validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + BooleanType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, + map_keys_check, + map_values_check, + nested_array_check, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_require_any_true, + check_require_if, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _names_check() -> Check: + return Check( + field="names", + name="required", + expr=check_required(F.col("names")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_required_check() -> Check: + return Check( + field="names.primary", + name="required", + expr=check_required(F.col("names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_string_min_length_check() -> Check: + return Check( + field="names.primary", + name="string_min_length", + expr=check_string_min_length(F.col("names.primary"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_stripped_check() -> Check: + return Check( + field="names.primary", + name="stripped", + expr=check_stripped(F.col("names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_common_key_check() -> Check: + return Check( + field="names.common{key}", + name="language_tag", + expr=map_keys_check( + "names.common", + lambda k: check_pattern( + k, + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_common_value_check() -> Check: + return Check( + field="names.common{value}", + name="stripped", + expr=map_values_check("names.common", lambda v: check_stripped(v)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_required_check() -> Check: + return Check( + field="names.rules[].value", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_string_min_length_check() -> Check: + return Check( + field="names.rules[].value", + name="string_min_length", + expr=array_check( + "names.rules", lambda el: check_string_min_length(el["value"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_stripped_check() -> Check: + return Check( + field="names.rules[].value", + name="stripped", + expr=array_check("names.rules", lambda el: check_stripped(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_required_check() -> Check: + return Check( + field="names.rules[].variant", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["variant"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_enum_check() -> Check: + return Check( + field="names.rules[].variant", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["variant"], ["common", "official", "alternate", "short"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_language_check() -> Check: + return Check( + field="names.rules[].language", + name="language_tag", + expr=array_check( + "names.rules", + lambda el: check_pattern( + el["language"], + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_required_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["mode"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_enum_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["perspectives"]["mode"], ["accepted_by", "disputed_by"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check() -> Check: + return Check( + field="names.rules[].perspectives.countries", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["countries"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_min_length_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_min_length", + name="array_min_length", + expr=array_check( + "names.rules", + lambda el: check_array_min_length(el["perspectives"]["countries"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_unique_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_unique", + name="struct_unique", + expr=array_check( + "names.rules", + lambda el: check_struct_unique(el["perspectives"]["countries"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check_1() -> Check: + return Check( + field="names.rules[].perspectives.countries[]", + name="country_code_alpha2", + expr=nested_array_check( + "names.rules", + lambda el: array_check( + el["perspectives"]["countries"], + lambda inner: check_pattern( + inner, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_length_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_length", + expr=array_check( + "names.rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_bounds", + expr=array_check( + "names.rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_order_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_order", + expr=array_check( + "names.rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_side_check() -> Check: + return Check( + field="names.rules[].side", + name="enum", + expr=array_check( + "names.rules", lambda el: check_enum(el["side"], ["left", "right"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type( + F.col("geometry"), GeometryType.MULTI_POLYGON, GeometryType.POLYGON + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["divisions"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["division_area"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _subtype_required_check() -> Check: + return Check( + field="subtype", + name="required", + expr=check_required(F.col("subtype")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _subtype_enum_check() -> Check: + return Check( + field="subtype", + name="enum", + expr=check_enum( + F.col("subtype"), + [ + "country", + "dependency", + "macroregion", + "region", + "macrocounty", + "county", + "localadmin", + "locality", + "borough", + "macrohood", + "neighborhood", + "microhood", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _class_required_check() -> Check: + return Check( + field="class", + name="required", + expr=check_required(F.col("class")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class"}), + ) + + +def _class_enum_check() -> Check: + return Check( + field="class", + name="enum", + expr=check_enum(F.col("class"), ["land", "maritime"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class"}), + ) + + +def _division_id_required_check() -> Check: + return Check( + field="division_id", + name="required", + expr=check_required(F.col("division_id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"division_id"}), + ) + + +def _division_id_string_min_length_check() -> Check: + return Check( + field="division_id", + name="string_min_length", + expr=check_string_min_length(F.col("division_id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"division_id"}), + ) + + +def _division_id_no_whitespace_check() -> Check: + return Check( + field="division_id", + name="no_whitespace", + expr=check_pattern( + F.col("division_id"), + "^\\S+\\z", + label="String without whitespace characters", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"division_id"}), + ) + + +def _country_required_check() -> Check: + return Check( + field="country", + name="required", + expr=check_required(F.col("country")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"country"}), + ) + + +def _country_country_code_alpha2_check() -> Check: + return Check( + field="country", + name="country_code_alpha2", + expr=check_pattern( + F.col("country"), "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"country"}), + ) + + +def _region_check() -> Check: + return Check( + field="region", + name="region_code", + expr=check_pattern( + F.col("region"), + "^[A-Z]{2}-[A-Z0-9]{1,3}\\z", + label="ISO 3166-2 subdivision code", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"region"}), + ) + + +def _admin_level_bounds_check() -> Check: + return Check( + field="admin_level_0", + name="bounds", + expr=check_bounds(F.col("admin_level"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level"}), + ) + + +def _admin_level_bounds_check_1() -> Check: + return Check( + field="admin_level_1", + name="bounds", + expr=check_bounds(F.col("admin_level"), le=16, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level"}), + ) + + +def _check_require_any_true_0_check() -> Check: + return Check( + field="require_any_true", + name="require_any_true", + expr=check_require_any_true( + [F.col("is_land") == F.lit(True), F.col("is_territorial") == F.lit(True)], + ["is_land", "is_territorial"], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"is_land", "is_territorial"}), + ) + + +def _check_require_if_1_check() -> Check: + return Check( + field="admin_level_required_0", + name="require_if", + expr=check_require_if( + F.col("admin_level"), F.col("subtype") == "county", "subtype = 'county'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_2_check() -> Check: + return Check( + field="admin_level_required_1", + name="require_if", + expr=check_require_if( + F.col("admin_level"), + F.col("subtype") == "macrocounty", + "subtype = 'macrocounty'", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_3_check() -> Check: + return Check( + field="admin_level_required_2", + name="require_if", + expr=check_require_if( + F.col("admin_level"), F.col("subtype") == "region", "subtype = 'region'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_4_check() -> Check: + return Check( + field="admin_level_required_3", + name="require_if", + expr=check_require_if( + F.col("admin_level"), + F.col("subtype") == "macroregion", + "subtype = 'macroregion'", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_5_check() -> Check: + return Check( + field="admin_level_required_4", + name="require_if", + expr=check_require_if( + F.col("admin_level"), + F.col("subtype") == "dependency", + "subtype = 'dependency'", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_6_check() -> Check: + return Check( + field="admin_level_required_5", + name="require_if", + expr=check_require_if( + F.col("admin_level"), F.col("subtype") == "country", "subtype = 'country'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def division_area_checks() -> list[Check]: + """All validation checks for division_area.""" + return [ + _names_check(), + _names_primary_required_check(), + _names_primary_string_min_length_check(), + _names_primary_stripped_check(), + _names_common_key_check(), + _names_common_value_check(), + _names_rules_value_required_check(), + _names_rules_value_string_min_length_check(), + _names_rules_value_stripped_check(), + _names_rules_variant_required_check(), + _names_rules_variant_enum_check(), + _names_rules_language_check(), + _names_rules_perspectives_mode_required_check(), + _names_rules_perspectives_mode_enum_check(), + _names_rules_perspectives_countries_check(), + _names_rules_perspectives_countries_min_length_check(), + _names_rules_perspectives_countries_unique_check(), + _names_rules_perspectives_countries_check_1(), + _names_rules_between_linear_range_length_check(), + _names_rules_between_linear_range_bounds_check(), + _names_rules_between_linear_range_order_check(), + _names_rules_side_check(), + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _subtype_required_check(), + _subtype_enum_check(), + _class_required_check(), + _class_enum_check(), + _division_id_required_check(), + _division_id_string_min_length_check(), + _division_id_no_whitespace_check(), + _country_required_check(), + _country_country_code_alpha2_check(), + _region_check(), + _admin_level_bounds_check(), + _admin_level_bounds_check_1(), + _check_require_any_true_0_check(), + _check_require_if_1_check(), + _check_require_if_2_check(), + _check_require_if_3_check(), + _check_require_if_4_check(), + _check_require_if_5_check(), + _check_require_if_6_check(), + ] + + +DIVISION_AREA_SCHEMA = StructType( + [ + StructField( + "names", + StructType( + [ + StructField("primary", StringType(), True), + StructField( + "common", MapType(StringType(), StringType(), True), True + ), + StructField( + "rules", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("variant", StringType(), True), + StructField("language", StringType(), True), + StructField( + "perspectives", + StructType( + [ + StructField("mode", StringType(), True), + StructField( + "countries", + ArrayType(StringType(), True), + True, + ), + ] + ), + True, + ), + StructField( + "between", ArrayType(DoubleType(), True), True + ), + StructField("side", StringType(), True), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("subtype", StringType(), True), + StructField("class", StringType(), True), + StructField("is_land", BooleanType(), True), + StructField("is_territorial", BooleanType(), True), + StructField("division_id", StringType(), True), + StructField("country", StringType(), True), + StructField("region", StringType(), True), + StructField("admin_level", IntegerType(), True), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = ( + GeometryType.MULTI_POLYGON, + GeometryType.POLYGON, +) + +ENTRY_POINT = "overture.schema.divisions:DivisionArea" + +PARTITIONS: dict[str, str] = {"theme": "divisions"} + +MODEL_VALIDATION = ModelValidation( + schema=DIVISION_AREA_SCHEMA, + checks=division_area_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/divisions/division_boundary.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/divisions/division_boundary.py new file mode 100644 index 000000000..094fccc6a --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/divisions/division_boundary.py @@ -0,0 +1,877 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Division Boundary validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + BooleanType, + DoubleType, + IntegerType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_max_length, + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_forbid_if, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_require_any_true, + check_require_if, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type( + F.col("geometry"), GeometryType.LINE_STRING, GeometryType.MULTI_LINE_STRING + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["divisions"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["division_boundary"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _subtype_required_check() -> Check: + return Check( + field="subtype", + name="required", + expr=check_required(F.col("subtype")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _subtype_enum_check() -> Check: + return Check( + field="subtype", + name="enum", + expr=check_enum( + F.col("subtype"), + [ + "country", + "dependency", + "macroregion", + "region", + "macrocounty", + "county", + "localadmin", + "locality", + "borough", + "macrohood", + "neighborhood", + "microhood", + ], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _class_required_check() -> Check: + return Check( + field="class", + name="required", + expr=check_required(F.col("class")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class"}), + ) + + +def _class_enum_check() -> Check: + return Check( + field="class", + name="enum", + expr=check_enum(F.col("class"), ["land", "maritime"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class"}), + ) + + +def _division_ids_check() -> Check: + return Check( + field="division_ids", + name="required", + expr=check_required(F.col("division_ids")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"division_ids"}), + ) + + +def _division_ids_min_length_check() -> Check: + return Check( + field="division_ids_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("division_ids"), 2), + shape=CheckShape.SCALAR, + read_columns=frozenset({"division_ids"}), + ) + + +def _division_ids_max_length_check() -> Check: + return Check( + field="division_ids_max_length", + name="array_max_length", + expr=check_array_max_length(F.col("division_ids"), 2), + shape=CheckShape.SCALAR, + read_columns=frozenset({"division_ids"}), + ) + + +def _division_ids_unique_check() -> Check: + return Check( + field="division_ids_unique", + name="struct_unique", + expr=check_struct_unique(F.col("division_ids")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"division_ids"}), + ) + + +def _division_ids_string_min_length_check() -> Check: + return Check( + field="division_ids[]", + name="string_min_length", + expr=array_check("division_ids", lambda el: check_string_min_length(el, 1)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"division_ids"}), + ) + + +def _division_ids_no_whitespace_check() -> Check: + return Check( + field="division_ids[]", + name="no_whitespace", + expr=array_check( + "division_ids", + lambda el: check_pattern( + el, "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"division_ids"}), + ) + + +def _country_check() -> Check: + return Check( + field="country", + name="country_code_alpha2", + expr=check_pattern( + F.col("country"), "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"country"}), + ) + + +def _region_check() -> Check: + return Check( + field="region", + name="region_code", + expr=check_pattern( + F.col("region"), + "^[A-Z]{2}-[A-Z0-9]{1,3}\\z", + label="ISO 3166-2 subdivision code", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"region"}), + ) + + +def _admin_level_bounds_check() -> Check: + return Check( + field="admin_level_0", + name="bounds", + expr=check_bounds(F.col("admin_level"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level"}), + ) + + +def _admin_level_bounds_check_1() -> Check: + return Check( + field="admin_level_1", + name="bounds", + expr=check_bounds(F.col("admin_level"), le=16, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level"}), + ) + + +def _perspectives_mode_required_check() -> Check: + return Check( + field="perspectives.mode", + name="required", + expr=F.when( + F.col("perspectives").isNotNull(), + check_required(F.col("perspectives.mode")), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"perspectives"}), + ) + + +def _perspectives_mode_enum_check() -> Check: + return Check( + field="perspectives.mode", + name="enum", + expr=check_enum(F.col("perspectives.mode"), ["accepted_by", "disputed_by"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"perspectives"}), + ) + + +def _perspectives_countries_check() -> Check: + return Check( + field="perspectives.countries", + name="required", + expr=F.when( + F.col("perspectives").isNotNull(), + check_required(F.col("perspectives.countries")), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"perspectives"}), + ) + + +def _perspectives_countries_min_length_check() -> Check: + return Check( + field="perspectives.countries_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("perspectives.countries"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"perspectives"}), + ) + + +def _perspectives_countries_unique_check() -> Check: + return Check( + field="perspectives.countries_unique", + name="struct_unique", + expr=check_struct_unique(F.col("perspectives.countries")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"perspectives"}), + ) + + +def _perspectives_countries_check_1() -> Check: + return Check( + field="perspectives.countries[]", + name="country_code_alpha2", + expr=array_check( + "perspectives.countries", + lambda el: check_pattern( + el, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"perspectives"}), + ) + + +def _check_require_any_true_0_check() -> Check: + return Check( + field="require_any_true", + name="require_any_true", + expr=check_require_any_true( + [F.col("is_land") == F.lit(True), F.col("is_territorial") == F.lit(True)], + ["is_land", "is_territorial"], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"is_land", "is_territorial"}), + ) + + +def _check_require_if_1_check() -> Check: + return Check( + field="admin_level_required_0", + name="require_if", + expr=check_require_if( + F.col("admin_level"), F.col("subtype") == "county", "subtype = 'county'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_2_check() -> Check: + return Check( + field="admin_level_required_1", + name="require_if", + expr=check_require_if( + F.col("admin_level"), + F.col("subtype") == "macrocounty", + "subtype = 'macrocounty'", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_3_check() -> Check: + return Check( + field="admin_level_required_2", + name="require_if", + expr=check_require_if( + F.col("admin_level"), F.col("subtype") == "region", "subtype = 'region'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_4_check() -> Check: + return Check( + field="admin_level_required_3", + name="require_if", + expr=check_require_if( + F.col("admin_level"), + F.col("subtype") == "macroregion", + "subtype = 'macroregion'", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_5_check() -> Check: + return Check( + field="admin_level_required_4", + name="require_if", + expr=check_require_if( + F.col("admin_level"), + F.col("subtype") == "dependency", + "subtype = 'dependency'", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_6_check() -> Check: + return Check( + field="admin_level_required_5", + name="require_if", + expr=check_require_if( + F.col("admin_level"), F.col("subtype") == "country", "subtype = 'country'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"admin_level", "subtype"}), + ) + + +def _check_require_if_7_check() -> Check: + return Check( + field="country_required", + name="require_if", + expr=check_require_if( + F.col("country"), F.col("subtype") != "country", "subtype != 'country'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"country", "subtype"}), + ) + + +def _check_forbid_if_8_check() -> Check: + return Check( + field="country_forbidden", + name="forbid_if", + expr=check_forbid_if( + F.col("country"), F.col("subtype") == "country", "subtype = 'country'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"country", "subtype"}), + ) + + +def division_boundary_checks() -> list[Check]: + """All validation checks for division_boundary.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _subtype_required_check(), + _subtype_enum_check(), + _class_required_check(), + _class_enum_check(), + _division_ids_check(), + _division_ids_min_length_check(), + _division_ids_max_length_check(), + _division_ids_unique_check(), + _division_ids_string_min_length_check(), + _division_ids_no_whitespace_check(), + _country_check(), + _region_check(), + _admin_level_bounds_check(), + _admin_level_bounds_check_1(), + _perspectives_mode_required_check(), + _perspectives_mode_enum_check(), + _perspectives_countries_check(), + _perspectives_countries_min_length_check(), + _perspectives_countries_unique_check(), + _perspectives_countries_check_1(), + _check_require_any_true_0_check(), + _check_require_if_1_check(), + _check_require_if_2_check(), + _check_require_if_3_check(), + _check_require_if_4_check(), + _check_require_if_5_check(), + _check_require_if_6_check(), + _check_require_if_7_check(), + _check_forbid_if_8_check(), + ] + + +DIVISION_BOUNDARY_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("subtype", StringType(), True), + StructField("class", StringType(), True), + StructField("is_land", BooleanType(), True), + StructField("is_territorial", BooleanType(), True), + StructField("division_ids", ArrayType(StringType(), True), True), + StructField("country", StringType(), True), + StructField("region", StringType(), True), + StructField("admin_level", IntegerType(), True), + StructField("is_disputed", BooleanType(), True), + StructField( + "perspectives", + StructType( + [ + StructField("mode", StringType(), True), + StructField("countries", ArrayType(StringType(), True), True), + ] + ), + True, + ), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = ( + GeometryType.LINE_STRING, + GeometryType.MULTI_LINE_STRING, +) + +ENTRY_POINT = "overture.schema.divisions:DivisionBoundary" + +PARTITIONS: dict[str, str] = {"theme": "divisions"} + +MODEL_VALIDATION = ModelValidation( + schema=DIVISION_BOUNDARY_SCHEMA, + checks=division_boundary_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/places/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/places/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/places/place.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/places/place.py new file mode 100644 index 000000000..e5c0138d5 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/places/place.py @@ -0,0 +1,1659 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Place validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, + map_keys_check, + map_values_check, + nested_array_check, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_email, + check_enum, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_required, + check_string_min_length, + check_stripped, + check_url_format, + check_url_length, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type(F.col("geometry"), GeometryType.POINT), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["places"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["place"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _operating_status_check() -> Check: + return Check( + field="operating_status", + name="enum", + expr=check_enum( + F.col("operating_status"), + ["open", "permanently_closed", "temporarily_closed"], + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"operating_status"}), + ) + + +def _categories_primary_required_check() -> Check: + return Check( + field="categories.primary", + name="required", + expr=F.when( + F.col("categories").isNotNull(), check_required(F.col("categories.primary")) + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"categories"}), + ) + + +def _categories_primary_snake_case_check() -> Check: + return Check( + field="categories.primary", + name="snake_case", + expr=check_pattern( + F.col("categories.primary"), + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"categories"}), + ) + + +def _categories_alternate_unique_check() -> Check: + return Check( + field="categories.alternate_unique", + name="struct_unique", + expr=check_struct_unique(F.col("categories.alternate")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"categories"}), + ) + + +def _categories_alternate_check() -> Check: + return Check( + field="categories.alternate[]", + name="snake_case", + expr=array_check( + "categories.alternate", + lambda el: check_pattern( + el, "^[a-z0-9]+(_[a-z0-9]+)*\\z", label="Category in snake_case format" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"categories"}), + ) + + +def _basic_category_check() -> Check: + return Check( + field="basic_category", + name="snake_case", + expr=check_pattern( + F.col("basic_category"), + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"basic_category"}), + ) + + +def _taxonomy_primary_required_check() -> Check: + return Check( + field="taxonomy.primary", + name="required", + expr=F.when( + F.col("taxonomy").isNotNull(), check_required(F.col("taxonomy.primary")) + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"taxonomy"}), + ) + + +def _taxonomy_primary_snake_case_check() -> Check: + return Check( + field="taxonomy.primary", + name="snake_case", + expr=check_pattern( + F.col("taxonomy.primary"), + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"taxonomy"}), + ) + + +def _taxonomy_hierarchy_check() -> Check: + return Check( + field="taxonomy.hierarchy", + name="required", + expr=F.when( + F.col("taxonomy").isNotNull(), check_required(F.col("taxonomy.hierarchy")) + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"taxonomy"}), + ) + + +def _taxonomy_hierarchy_min_length_check() -> Check: + return Check( + field="taxonomy.hierarchy_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("taxonomy.hierarchy"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"taxonomy"}), + ) + + +def _taxonomy_hierarchy_unique_check() -> Check: + return Check( + field="taxonomy.hierarchy_unique", + name="struct_unique", + expr=check_struct_unique(F.col("taxonomy.hierarchy")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"taxonomy"}), + ) + + +def _taxonomy_hierarchy_check_1() -> Check: + return Check( + field="taxonomy.hierarchy[]", + name="snake_case", + expr=array_check( + "taxonomy.hierarchy", + lambda el: check_pattern( + el, "^[a-z0-9]+(_[a-z0-9]+)*\\z", label="Category in snake_case format" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"taxonomy"}), + ) + + +def _taxonomy_alternates_min_length_check() -> Check: + return Check( + field="taxonomy.alternates_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("taxonomy.alternates"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"taxonomy"}), + ) + + +def _taxonomy_alternates_unique_check() -> Check: + return Check( + field="taxonomy.alternates_unique", + name="struct_unique", + expr=check_struct_unique(F.col("taxonomy.alternates")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"taxonomy"}), + ) + + +def _taxonomy_alternates_check() -> Check: + return Check( + field="taxonomy.alternates[]", + name="snake_case", + expr=array_check( + "taxonomy.alternates", + lambda el: check_pattern( + el, "^[a-z0-9]+(_[a-z0-9]+)*\\z", label="Category in snake_case format" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"taxonomy"}), + ) + + +def _confidence_bounds_check() -> Check: + return Check( + field="confidence_0", + name="bounds", + expr=check_bounds(F.col("confidence"), ge=0.0), + shape=CheckShape.SCALAR, + read_columns=frozenset({"confidence"}), + ) + + +def _confidence_bounds_check_1() -> Check: + return Check( + field="confidence_1", + name="bounds", + expr=check_bounds(F.col("confidence"), le=1.0), + shape=CheckShape.SCALAR, + read_columns=frozenset({"confidence"}), + ) + + +def _websites_min_length_check() -> Check: + return Check( + field="websites_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("websites"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"websites"}), + ) + + +def _websites_unique_check() -> Check: + return Check( + field="websites_unique", + name="struct_unique", + expr=check_struct_unique(F.col("websites")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"websites"}), + ) + + +def _websites_url_format_check() -> Check: + return Check( + field="websites[]", + name="url_format", + expr=array_check("websites", lambda el: check_url_format(el)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"websites"}), + ) + + +def _websites_url_length_check() -> Check: + return Check( + field="websites[]", + name="url_length", + expr=array_check("websites", lambda el: check_url_length(el)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"websites"}), + ) + + +def _socials_min_length_check() -> Check: + return Check( + field="socials_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("socials"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"socials"}), + ) + + +def _socials_unique_check() -> Check: + return Check( + field="socials_unique", + name="struct_unique", + expr=check_struct_unique(F.col("socials")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"socials"}), + ) + + +def _socials_url_format_check() -> Check: + return Check( + field="socials[]", + name="url_format", + expr=array_check("socials", lambda el: check_url_format(el)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"socials"}), + ) + + +def _socials_url_length_check() -> Check: + return Check( + field="socials[]", + name="url_length", + expr=array_check("socials", lambda el: check_url_length(el)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"socials"}), + ) + + +def _emails_min_length_check() -> Check: + return Check( + field="emails_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("emails"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"emails"}), + ) + + +def _emails_unique_check() -> Check: + return Check( + field="emails_unique", + name="struct_unique", + expr=check_struct_unique(F.col("emails")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"emails"}), + ) + + +def _emails_check() -> Check: + return Check( + field="emails[]", + name="email", + expr=array_check("emails", lambda el: check_email(el)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"emails"}), + ) + + +def _phones_min_length_check() -> Check: + return Check( + field="phones_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("phones"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"phones"}), + ) + + +def _phones_unique_check() -> Check: + return Check( + field="phones_unique", + name="struct_unique", + expr=check_struct_unique(F.col("phones")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"phones"}), + ) + + +def _phones_check() -> Check: + return Check( + field="phones[]", + name="phone_number", + expr=array_check( + "phones", + lambda el: check_pattern( + el, + "^\\+\\d{1,3}[\\s\\-\\(\\)0-9]+\\z", + label="International phone number (+ followed by country code and number)", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"phones"}), + ) + + +def _brand_names_primary_required_check() -> Check: + return Check( + field="brand.names.primary", + name="required", + expr=F.when( + F.col("brand.names").isNotNull(), + check_required(F.col("brand.names.primary")), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_primary_string_min_length_check() -> Check: + return Check( + field="brand.names.primary", + name="string_min_length", + expr=check_string_min_length(F.col("brand.names.primary"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_primary_stripped_check() -> Check: + return Check( + field="brand.names.primary", + name="stripped", + expr=check_stripped(F.col("brand.names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_common_key_check() -> Check: + return Check( + field="brand.names.common{key}", + name="language_tag", + expr=map_keys_check( + "brand.names.common", + lambda k: check_pattern( + k, + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_common_value_check() -> Check: + return Check( + field="brand.names.common{value}", + name="stripped", + expr=map_values_check("brand.names.common", lambda v: check_stripped(v)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_value_required_check() -> Check: + return Check( + field="brand.names.rules[].value", + name="required", + expr=array_check("brand.names.rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_value_string_min_length_check() -> Check: + return Check( + field="brand.names.rules[].value", + name="string_min_length", + expr=array_check( + "brand.names.rules", lambda el: check_string_min_length(el["value"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_value_stripped_check() -> Check: + return Check( + field="brand.names.rules[].value", + name="stripped", + expr=array_check("brand.names.rules", lambda el: check_stripped(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_variant_required_check() -> Check: + return Check( + field="brand.names.rules[].variant", + name="required", + expr=array_check("brand.names.rules", lambda el: check_required(el["variant"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_variant_enum_check() -> Check: + return Check( + field="brand.names.rules[].variant", + name="enum", + expr=array_check( + "brand.names.rules", + lambda el: check_enum( + el["variant"], ["common", "official", "alternate", "short"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_language_check() -> Check: + return Check( + field="brand.names.rules[].language", + name="language_tag", + expr=array_check( + "brand.names.rules", + lambda el: check_pattern( + el["language"], + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_perspectives_mode_required_check() -> Check: + return Check( + field="brand.names.rules[].perspectives.mode", + name="required", + expr=array_check( + "brand.names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["mode"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_perspectives_mode_enum_check() -> Check: + return Check( + field="brand.names.rules[].perspectives.mode", + name="enum", + expr=array_check( + "brand.names.rules", + lambda el: check_enum( + el["perspectives"]["mode"], ["accepted_by", "disputed_by"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_perspectives_countries_check() -> Check: + return Check( + field="brand.names.rules[].perspectives.countries", + name="required", + expr=array_check( + "brand.names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["countries"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_perspectives_countries_min_length_check() -> Check: + return Check( + field="brand.names.rules[].perspectives.countries_min_length", + name="array_min_length", + expr=array_check( + "brand.names.rules", + lambda el: check_array_min_length(el["perspectives"]["countries"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_perspectives_countries_unique_check() -> Check: + return Check( + field="brand.names.rules[].perspectives.countries_unique", + name="struct_unique", + expr=array_check( + "brand.names.rules", + lambda el: check_struct_unique(el["perspectives"]["countries"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_perspectives_countries_check_1() -> Check: + return Check( + field="brand.names.rules[].perspectives.countries[]", + name="country_code_alpha2", + expr=nested_array_check( + "brand.names.rules", + lambda el: array_check( + el["perspectives"]["countries"], + lambda inner: check_pattern( + inner, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_between_linear_range_length_check() -> Check: + return Check( + field="brand.names.rules[].between", + name="linear_range_length", + expr=array_check( + "brand.names.rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="brand.names.rules[].between", + name="linear_range_bounds", + expr=array_check( + "brand.names.rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_between_linear_range_order_check() -> Check: + return Check( + field="brand.names.rules[].between", + name="linear_range_order", + expr=array_check( + "brand.names.rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_names_rules_side_check() -> Check: + return Check( + field="brand.names.rules[].side", + name="enum", + expr=array_check( + "brand.names.rules", lambda el: check_enum(el["side"], ["left", "right"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"brand"}), + ) + + +def _brand_wikidata_check() -> Check: + return Check( + field="brand.wikidata", + name="wikidata_id", + expr=check_pattern( + F.col("brand.wikidata"), + "^Q\\d+\\z", + label="Wikidata identifier (Q followed by digits)", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"brand"}), + ) + + +def _addresses_min_length_check() -> Check: + return Check( + field="addresses_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("addresses"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"addresses"}), + ) + + +def _addresses_region_check() -> Check: + return Check( + field="addresses[].region", + name="region_code", + expr=array_check( + "addresses", + lambda el: check_pattern( + el["region"], + "^[A-Z]{2}-[A-Z0-9]{1,3}\\z", + label="ISO 3166-2 subdivision code", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"addresses"}), + ) + + +def _addresses_country_check() -> Check: + return Check( + field="addresses[].country", + name="country_code_alpha2", + expr=array_check( + "addresses", + lambda el: check_pattern( + el["country"], "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"addresses"}), + ) + + +def _names_primary_required_check() -> Check: + return Check( + field="names.primary", + name="required", + expr=F.when(F.col("names").isNotNull(), check_required(F.col("names.primary"))), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_string_min_length_check() -> Check: + return Check( + field="names.primary", + name="string_min_length", + expr=check_string_min_length(F.col("names.primary"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_stripped_check() -> Check: + return Check( + field="names.primary", + name="stripped", + expr=check_stripped(F.col("names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_common_key_check() -> Check: + return Check( + field="names.common{key}", + name="language_tag", + expr=map_keys_check( + "names.common", + lambda k: check_pattern( + k, + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_common_value_check() -> Check: + return Check( + field="names.common{value}", + name="stripped", + expr=map_values_check("names.common", lambda v: check_stripped(v)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_required_check() -> Check: + return Check( + field="names.rules[].value", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_string_min_length_check() -> Check: + return Check( + field="names.rules[].value", + name="string_min_length", + expr=array_check( + "names.rules", lambda el: check_string_min_length(el["value"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_stripped_check() -> Check: + return Check( + field="names.rules[].value", + name="stripped", + expr=array_check("names.rules", lambda el: check_stripped(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_required_check() -> Check: + return Check( + field="names.rules[].variant", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["variant"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_enum_check() -> Check: + return Check( + field="names.rules[].variant", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["variant"], ["common", "official", "alternate", "short"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_language_check() -> Check: + return Check( + field="names.rules[].language", + name="language_tag", + expr=array_check( + "names.rules", + lambda el: check_pattern( + el["language"], + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_required_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["mode"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_enum_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["perspectives"]["mode"], ["accepted_by", "disputed_by"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check() -> Check: + return Check( + field="names.rules[].perspectives.countries", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["countries"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_min_length_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_min_length", + name="array_min_length", + expr=array_check( + "names.rules", + lambda el: check_array_min_length(el["perspectives"]["countries"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_unique_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_unique", + name="struct_unique", + expr=array_check( + "names.rules", + lambda el: check_struct_unique(el["perspectives"]["countries"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check_1() -> Check: + return Check( + field="names.rules[].perspectives.countries[]", + name="country_code_alpha2", + expr=nested_array_check( + "names.rules", + lambda el: array_check( + el["perspectives"]["countries"], + lambda inner: check_pattern( + inner, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_length_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_length", + expr=array_check( + "names.rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_bounds", + expr=array_check( + "names.rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_order_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_order", + expr=array_check( + "names.rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_side_check() -> Check: + return Check( + field="names.rules[].side", + name="enum", + expr=array_check( + "names.rules", lambda el: check_enum(el["side"], ["left", "right"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def place_checks() -> list[Check]: + """All validation checks for place.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _operating_status_check(), + _categories_primary_required_check(), + _categories_primary_snake_case_check(), + _categories_alternate_unique_check(), + _categories_alternate_check(), + _basic_category_check(), + _taxonomy_primary_required_check(), + _taxonomy_primary_snake_case_check(), + _taxonomy_hierarchy_check(), + _taxonomy_hierarchy_min_length_check(), + _taxonomy_hierarchy_unique_check(), + _taxonomy_hierarchy_check_1(), + _taxonomy_alternates_min_length_check(), + _taxonomy_alternates_unique_check(), + _taxonomy_alternates_check(), + _confidence_bounds_check(), + _confidence_bounds_check_1(), + _websites_min_length_check(), + _websites_unique_check(), + _websites_url_format_check(), + _websites_url_length_check(), + _socials_min_length_check(), + _socials_unique_check(), + _socials_url_format_check(), + _socials_url_length_check(), + _emails_min_length_check(), + _emails_unique_check(), + _emails_check(), + _phones_min_length_check(), + _phones_unique_check(), + _phones_check(), + _brand_names_primary_required_check(), + _brand_names_primary_string_min_length_check(), + _brand_names_primary_stripped_check(), + _brand_names_common_key_check(), + _brand_names_common_value_check(), + _brand_names_rules_value_required_check(), + _brand_names_rules_value_string_min_length_check(), + _brand_names_rules_value_stripped_check(), + _brand_names_rules_variant_required_check(), + _brand_names_rules_variant_enum_check(), + _brand_names_rules_language_check(), + _brand_names_rules_perspectives_mode_required_check(), + _brand_names_rules_perspectives_mode_enum_check(), + _brand_names_rules_perspectives_countries_check(), + _brand_names_rules_perspectives_countries_min_length_check(), + _brand_names_rules_perspectives_countries_unique_check(), + _brand_names_rules_perspectives_countries_check_1(), + _brand_names_rules_between_linear_range_length_check(), + _brand_names_rules_between_linear_range_bounds_check(), + _brand_names_rules_between_linear_range_order_check(), + _brand_names_rules_side_check(), + _brand_wikidata_check(), + _addresses_min_length_check(), + _addresses_region_check(), + _addresses_country_check(), + _names_primary_required_check(), + _names_primary_string_min_length_check(), + _names_primary_stripped_check(), + _names_common_key_check(), + _names_common_value_check(), + _names_rules_value_required_check(), + _names_rules_value_string_min_length_check(), + _names_rules_value_stripped_check(), + _names_rules_variant_required_check(), + _names_rules_variant_enum_check(), + _names_rules_language_check(), + _names_rules_perspectives_mode_required_check(), + _names_rules_perspectives_mode_enum_check(), + _names_rules_perspectives_countries_check(), + _names_rules_perspectives_countries_min_length_check(), + _names_rules_perspectives_countries_unique_check(), + _names_rules_perspectives_countries_check_1(), + _names_rules_between_linear_range_length_check(), + _names_rules_between_linear_range_bounds_check(), + _names_rules_between_linear_range_order_check(), + _names_rules_side_check(), + ] + + +PLACE_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("operating_status", StringType(), True), + StructField( + "categories", + StructType( + [ + StructField("primary", StringType(), True), + StructField("alternate", ArrayType(StringType(), True), True), + ] + ), + True, + ), + StructField("basic_category", StringType(), True), + StructField( + "taxonomy", + StructType( + [ + StructField("primary", StringType(), True), + StructField("hierarchy", ArrayType(StringType(), True), True), + StructField("alternates", ArrayType(StringType(), True), True), + ] + ), + True, + ), + StructField("confidence", DoubleType(), True), + StructField("websites", ArrayType(StringType(), True), True), + StructField("socials", ArrayType(StringType(), True), True), + StructField("emails", ArrayType(StringType(), True), True), + StructField("phones", ArrayType(StringType(), True), True), + StructField( + "brand", + StructType( + [ + StructField( + "names", + StructType( + [ + StructField("primary", StringType(), True), + StructField( + "common", + MapType(StringType(), StringType(), True), + True, + ), + StructField( + "rules", + ArrayType( + StructType( + [ + StructField( + "value", StringType(), True + ), + StructField( + "variant", StringType(), True + ), + StructField( + "language", StringType(), True + ), + StructField( + "perspectives", + StructType( + [ + StructField( + "mode", + StringType(), + True, + ), + StructField( + "countries", + ArrayType( + StringType(), True + ), + True, + ), + ] + ), + True, + ), + StructField( + "between", + ArrayType(DoubleType(), True), + True, + ), + StructField("side", StringType(), True), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + StructField("wikidata", StringType(), True), + ] + ), + True, + ), + StructField( + "addresses", + ArrayType( + StructType( + [ + StructField("freeform", StringType(), True), + StructField("locality", StringType(), True), + StructField("postcode", StringType(), True), + StructField("region", StringType(), True), + StructField("country", StringType(), True), + ] + ), + True, + ), + True, + ), + StructField( + "names", + StructType( + [ + StructField("primary", StringType(), True), + StructField( + "common", MapType(StringType(), StringType(), True), True + ), + StructField( + "rules", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("variant", StringType(), True), + StructField("language", StringType(), True), + StructField( + "perspectives", + StructType( + [ + StructField("mode", StringType(), True), + StructField( + "countries", + ArrayType(StringType(), True), + True, + ), + ] + ), + True, + ), + StructField( + "between", ArrayType(DoubleType(), True), True + ), + StructField("side", StringType(), True), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = (GeometryType.POINT,) + +ENTRY_POINT = "overture.schema.places:Place" + +PARTITIONS: dict[str, str] = {"theme": "places"} + +MODEL_VALIDATION = ModelValidation( + schema=PLACE_SCHEMA, + checks=place_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/transportation/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/transportation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/transportation/connector.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/transportation/connector.py new file mode 100644 index 000000000..b7f8b5dcb --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/transportation/connector.py @@ -0,0 +1,466 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Connector validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + DoubleType, + IntegerType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type(F.col("geometry"), GeometryType.POINT), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["transportation"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["connector"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def connector_checks() -> list[Check]: + """All validation checks for connector.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + ] + + +CONNECTOR_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = (GeometryType.POINT,) + +ENTRY_POINT = "overture.schema.transportation:Connector" + +PARTITIONS: dict[str, str] = {"theme": "transportation"} + +MODEL_VALIDATION = ModelValidation( + schema=CONNECTOR_SCHEMA, + checks=connector_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/transportation/segment.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/transportation/segment.py new file mode 100644 index 000000000..5ef1d437e --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/transportation/segment.py @@ -0,0 +1,5197 @@ +# This file is auto-generated by overture-schema-codegen. Do not edit. + +"""Segment validation expression builders.""" + +from __future__ import annotations + +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + BinaryType, + BooleanType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + +from overture.schema.pyspark.check import Check, CheckShape, ModelValidation +from overture.schema.pyspark.expressions._schema_structs import ( + BBOX_STRUCT, +) +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, + map_keys_check, + map_values_check, + nested_array_check, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_enum, + check_forbid_if, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_pattern, + check_require_any_of, + check_require_if, + check_required, + check_string_min_length, + check_stripped, +) +from overture.schema.system.primitive import GeometryType + + +def _id_required_check() -> Check: + return Check( + field="id", + name="required", + expr=check_required(F.col("id")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_string_min_length_check() -> Check: + return Check( + field="id", + name="string_min_length", + expr=check_string_min_length(F.col("id"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _id_no_whitespace_check() -> Check: + return Check( + field="id", + name="no_whitespace", + expr=check_pattern( + F.col("id"), "^\\S+\\z", label="String without whitespace characters" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + + +def _bbox_bbox_completeness_check() -> Check: + return Check( + field="bbox", + name="bbox_completeness", + expr=check_bbox_completeness(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_ordering_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_ordering", + expr=check_bbox_lat_ordering(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _bbox_bbox_lat_range_check() -> Check: + return Check( + field="bbox", + name="bbox_lat_range", + expr=check_bbox_lat_range(F.col("bbox")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"bbox"}), + ) + + +def _geometry_required_check() -> Check: + return Check( + field="geometry", + name="required", + expr=check_required(F.col("geometry")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _geometry_geometry_type_check() -> Check: + return Check( + field="geometry", + name="geometry_type", + expr=check_geometry_type(F.col("geometry"), GeometryType.LINE_STRING), + shape=CheckShape.SCALAR, + read_columns=frozenset({"geometry"}), + ) + + +def _theme_required_check() -> Check: + return Check( + field="theme", + name="required", + expr=check_required(F.col("theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _theme_enum_check() -> Check: + return Check( + field="theme", + name="enum", + expr=check_enum(F.col("theme"), ["transportation"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ) + + +def _type_required_check() -> Check: + return Check( + field="type", + name="required", + expr=check_required(F.col("type")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _type_enum_check() -> Check: + return Check( + field="type", + name="enum", + expr=check_enum(F.col("type"), ["segment"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"type"}), + ) + + +def _version_required_check() -> Check: + return Check( + field="version", + name="required", + expr=check_required(F.col("version")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _version_bounds_check() -> Check: + return Check( + field="version", + name="bounds", + expr=check_bounds(F.col("version"), ge=0, check_nan=False), + shape=CheckShape.SCALAR, + read_columns=frozenset({"version"}), + ) + + +def _sources_min_length_check() -> Check: + return Check( + field="sources_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("sources"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_unique_check() -> Check: + return Check( + field="sources_unique", + name="struct_unique", + expr=check_struct_unique(F.col("sources")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_required_check() -> Check: + return Check( + field="sources[].property", + name="required", + expr=array_check("sources", lambda el: check_required(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_property_json_pointer_check() -> Check: + return Check( + field="sources[].property", + name="json_pointer", + expr=array_check("sources", lambda el: check_json_pointer(el["property"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_dataset_check() -> Check: + return Check( + field="sources[].dataset", + name="required", + expr=array_check("sources", lambda el: check_required(el["dataset"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_license_check() -> Check: + return Check( + field="sources[].license", + name="stripped", + expr=array_check("sources", lambda el: check_stripped(el["license"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check() -> Check: + return Check( + field="sources[].confidence_0", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_confidence_bounds_check_1() -> Check: + return Check( + field="sources[].confidence_1", + name="bounds", + expr=array_check("sources", lambda el: check_bounds(el["confidence"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_string_min_length_check() -> Check: + return Check( + field="sources[].provider", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["provider"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_provider_snake_case_check() -> Check: + return Check( + field="sources[].provider", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["provider"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_string_min_length_check() -> Check: + return Check( + field="sources[].resource", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["resource"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_resource_snake_case_check() -> Check: + return Check( + field="sources[].resource", + name="snake_case", + expr=array_check( + "sources", + lambda el: check_pattern( + el["resource"], + "^[a-z0-9]+(_[a-z0-9]+)*\\z", + label="Category in snake_case format", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_string_min_length_check() -> Check: + return Check( + field="sources[].version", + name="string_min_length", + expr=array_check( + "sources", lambda el: check_string_min_length(el["version"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_version_no_whitespace_check() -> Check: + return Check( + field="sources[].version", + name="no_whitespace", + expr=array_check( + "sources", + lambda el: check_pattern( + el["version"], "^\\S+\\z", label="String without whitespace characters" + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_length_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_length", + expr=array_check( + "sources", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_bounds_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_bounds", + expr=array_check( + "sources", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _sources_between_linear_range_order_check() -> Check: + return Check( + field="sources[].between", + name="linear_range_order", + expr=array_check("sources", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ) + + +def _subtype_required_check() -> Check: + return Check( + field="subtype", + name="required", + expr=check_required(F.col("subtype")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _subtype_enum_check() -> Check: + return Check( + field="subtype", + name="enum", + expr=check_enum(F.col("subtype"), ["road", "rail", "water"]), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + + +def _access_restrictions_min_length_check() -> Check: + return Check( + field="access_restrictions_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("access_restrictions"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_unique_check() -> Check: + return Check( + field="access_restrictions_unique", + name="struct_unique", + expr=check_struct_unique(F.col("access_restrictions")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_access_type_required_check() -> Check: + return Check( + field="access_restrictions[].access_type", + name="required", + expr=array_check( + "access_restrictions", lambda el: check_required(el["access_type"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_access_type_enum_check() -> Check: + return Check( + field="access_restrictions[].access_type", + name="enum", + expr=array_check( + "access_restrictions", + lambda el: check_enum( + el["access_type"], ["allowed", "denied", "designated"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_between_linear_range_length_check() -> Check: + return Check( + field="access_restrictions[].between", + name="linear_range_length", + expr=array_check( + "access_restrictions", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_between_linear_range_bounds_check() -> Check: + return Check( + field="access_restrictions[].between", + name="linear_range_bounds", + expr=array_check( + "access_restrictions", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_between_linear_range_order_check() -> Check: + return Check( + field="access_restrictions[].between", + name="linear_range_order", + expr=array_check( + "access_restrictions", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_heading_check() -> Check: + return Check( + field="access_restrictions[].when.heading", + name="enum", + expr=array_check( + "access_restrictions", + lambda el: check_enum(el["when"]["heading"], ["forward", "backward"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_mode_min_length_check() -> Check: + return Check( + field="access_restrictions[].when.mode_min_length", + name="array_min_length", + expr=array_check( + "access_restrictions", + lambda el: check_array_min_length(el["when"]["mode"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_mode_unique_check() -> Check: + return Check( + field="access_restrictions[].when.mode_unique", + name="struct_unique", + expr=array_check( + "access_restrictions", lambda el: check_struct_unique(el["when"]["mode"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_mode_check() -> Check: + return Check( + field="access_restrictions[].when.mode[]", + name="enum", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["mode"], + lambda inner: check_enum( + inner, + [ + "vehicle", + "motor_vehicle", + "car", + "truck", + "motorcycle", + "foot", + "bicycle", + "bus", + "hgv", + "hov", + "emergency", + ], + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_using_min_length_check() -> Check: + return Check( + field="access_restrictions[].when.using_min_length", + name="array_min_length", + expr=array_check( + "access_restrictions", + lambda el: check_array_min_length(el["when"]["using"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_using_unique_check() -> Check: + return Check( + field="access_restrictions[].when.using_unique", + name="struct_unique", + expr=array_check( + "access_restrictions", lambda el: check_struct_unique(el["when"]["using"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_using_check() -> Check: + return Check( + field="access_restrictions[].when.using[]", + name="enum", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["using"], + lambda inner: check_enum( + inner, + [ + "as_customer", + "at_destination", + "to_deliver", + "to_farm", + "for_forestry", + ], + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_recognized_min_length_check() -> Check: + return Check( + field="access_restrictions[].when.recognized_min_length", + name="array_min_length", + expr=array_check( + "access_restrictions", + lambda el: check_array_min_length(el["when"]["recognized"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_recognized_unique_check() -> Check: + return Check( + field="access_restrictions[].when.recognized_unique", + name="struct_unique", + expr=array_check( + "access_restrictions", + lambda el: check_struct_unique(el["when"]["recognized"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_recognized_check() -> Check: + return Check( + field="access_restrictions[].when.recognized[]", + name="enum", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["recognized"], + lambda inner: check_enum( + inner, + [ + "as_permitted", + "as_private", + "as_disabled", + "as_employee", + "as_student", + ], + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_min_length_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle_min_length", + name="array_min_length", + expr=array_check( + "access_restrictions", + lambda el: check_array_min_length(el["when"]["vehicle"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_unique_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle_unique", + name="struct_unique", + expr=array_check( + "access_restrictions", lambda el: check_struct_unique(el["when"]["vehicle"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_dimension_required_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].dimension", + name="required", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], lambda inner: check_required(inner["dimension"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_dimension_enum_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].dimension", + name="enum", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_enum( + inner["dimension"], + ["axle_count", "height", "length", "weight", "width"], + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_comparison_required_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].comparison", + name="required", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], lambda inner: check_required(inner["comparison"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_comparison_enum_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].comparison", + name="enum", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_enum( + inner["comparison"], + [ + "greater_than", + "greater_than_equal", + "equal", + "less_than", + "less_than_equal", + ], + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_value_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].value_0", + name="required", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["axle_count"]), + check_required(inner["value"]), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_value_required_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].value_1", + name="required", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["height", "length", "weight", "width"]), + check_required(inner["value"]), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_value_bounds_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].value", + name="bounds", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["height", "length", "weight", "width"]), + check_bounds(inner["value"], ge=0.0), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_unit_required_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].unit_0", + name="required", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["height", "length", "width"]), + check_required(inner["unit"]), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_unit_enum_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].unit_0", + name="enum", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["height", "length", "width"]), + check_enum( + inner["unit"], ["in", "ft", "yd", "mi", "cm", "m", "km"] + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_unit_required_check_1() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].unit_1", + name="required", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["weight"]), check_required(inner["unit"]) + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_unit_enum_check_1() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].unit_1", + name="enum", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["weight"]), + check_enum(inner["unit"], ["oz", "lb", "st", "lt", "g", "kg", "t"]), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _connectors_min_length_check() -> Check: + return Check( + field="connectors_min_length", + name="array_min_length", + expr=check_array_min_length(F.col("connectors"), 2), + shape=CheckShape.SCALAR, + read_columns=frozenset({"connectors"}), + ) + + +def _connectors_unique_check() -> Check: + return Check( + field="connectors_unique", + name="struct_unique", + expr=check_struct_unique(F.col("connectors")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"connectors"}), + ) + + +def _connectors_connector_id_required_check() -> Check: + return Check( + field="connectors[].connector_id", + name="required", + expr=array_check("connectors", lambda el: check_required(el["connector_id"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"connectors"}), + ) + + +def _connectors_connector_id_string_min_length_check() -> Check: + return Check( + field="connectors[].connector_id", + name="string_min_length", + expr=array_check( + "connectors", lambda el: check_string_min_length(el["connector_id"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"connectors"}), + ) + + +def _connectors_connector_id_no_whitespace_check() -> Check: + return Check( + field="connectors[].connector_id", + name="no_whitespace", + expr=array_check( + "connectors", + lambda el: check_pattern( + el["connector_id"], + "^\\S+\\z", + label="String without whitespace characters", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"connectors"}), + ) + + +def _connectors_at_bounds_check() -> Check: + return Check( + field="connectors[].at_0", + name="bounds", + expr=array_check("connectors", lambda el: check_bounds(el["at"], ge=0.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"connectors"}), + ) + + +def _connectors_at_bounds_check_1() -> Check: + return Check( + field="connectors[].at_1", + name="bounds", + expr=array_check("connectors", lambda el: check_bounds(el["at"], le=1.0)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"connectors"}), + ) + + +def _level_rules_value_check() -> Check: + return Check( + field="level_rules[].value", + name="required", + expr=array_check("level_rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"level_rules"}), + ) + + +def _level_rules_between_linear_range_length_check() -> Check: + return Check( + field="level_rules[].between", + name="linear_range_length", + expr=array_check( + "level_rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"level_rules"}), + ) + + +def _level_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="level_rules[].between", + name="linear_range_bounds", + expr=array_check( + "level_rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"level_rules"}), + ) + + +def _level_rules_between_linear_range_order_check() -> Check: + return Check( + field="level_rules[].between", + name="linear_range_order", + expr=array_check( + "level_rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"level_rules"}), + ) + + +def _routes_name_string_min_length_check() -> Check: + return Check( + field="routes[].name", + name="string_min_length", + expr=array_check("routes", lambda el: check_string_min_length(el["name"], 1)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"routes"}), + ) + + +def _routes_name_stripped_check() -> Check: + return Check( + field="routes[].name", + name="stripped", + expr=array_check("routes", lambda el: check_stripped(el["name"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"routes"}), + ) + + +def _routes_network_string_min_length_check() -> Check: + return Check( + field="routes[].network", + name="string_min_length", + expr=array_check( + "routes", lambda el: check_string_min_length(el["network"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"routes"}), + ) + + +def _routes_network_stripped_check() -> Check: + return Check( + field="routes[].network", + name="stripped", + expr=array_check("routes", lambda el: check_stripped(el["network"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"routes"}), + ) + + +def _routes_ref_string_min_length_check() -> Check: + return Check( + field="routes[].ref", + name="string_min_length", + expr=array_check("routes", lambda el: check_string_min_length(el["ref"], 1)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"routes"}), + ) + + +def _routes_ref_stripped_check() -> Check: + return Check( + field="routes[].ref", + name="stripped", + expr=array_check("routes", lambda el: check_stripped(el["ref"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"routes"}), + ) + + +def _routes_symbol_string_min_length_check() -> Check: + return Check( + field="routes[].symbol", + name="string_min_length", + expr=array_check("routes", lambda el: check_string_min_length(el["symbol"], 1)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"routes"}), + ) + + +def _routes_symbol_stripped_check() -> Check: + return Check( + field="routes[].symbol", + name="stripped", + expr=array_check("routes", lambda el: check_stripped(el["symbol"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"routes"}), + ) + + +def _routes_wikidata_check() -> Check: + return Check( + field="routes[].wikidata", + name="wikidata_id", + expr=array_check( + "routes", + lambda el: check_pattern( + el["wikidata"], + "^Q\\d+\\z", + label="Wikidata identifier (Q followed by digits)", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"routes"}), + ) + + +def _routes_between_linear_range_length_check() -> Check: + return Check( + field="routes[].between", + name="linear_range_length", + expr=array_check("routes", lambda el: check_linear_range_length(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"routes"}), + ) + + +def _routes_between_linear_range_bounds_check() -> Check: + return Check( + field="routes[].between", + name="linear_range_bounds", + expr=array_check("routes", lambda el: check_linear_range_bounds(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"routes"}), + ) + + +def _routes_between_linear_range_order_check() -> Check: + return Check( + field="routes[].between", + name="linear_range_order", + expr=array_check("routes", lambda el: check_linear_range_order(el["between"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"routes"}), + ) + + +def _subclass_rules_value_required_check() -> Check: + return Check( + field="subclass_rules[].value", + name="required", + expr=array_check("subclass_rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"subclass_rules"}), + ) + + +def _subclass_rules_value_enum_check() -> Check: + return Check( + field="subclass_rules[].value", + name="enum", + expr=array_check( + "subclass_rules", + lambda el: check_enum( + el["value"], + [ + "link", + "sidewalk", + "crosswalk", + "parking_aisle", + "driveway", + "alley", + "cycle_crossing", + ], + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"subclass_rules"}), + ) + + +def _subclass_rules_between_linear_range_length_check() -> Check: + return Check( + field="subclass_rules[].between", + name="linear_range_length", + expr=array_check( + "subclass_rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"subclass_rules"}), + ) + + +def _subclass_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="subclass_rules[].between", + name="linear_range_bounds", + expr=array_check( + "subclass_rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"subclass_rules"}), + ) + + +def _subclass_rules_between_linear_range_order_check() -> Check: + return Check( + field="subclass_rules[].between", + name="linear_range_order", + expr=array_check( + "subclass_rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"subclass_rules"}), + ) + + +def _names_primary_required_check() -> Check: + return Check( + field="names.primary", + name="required", + expr=F.when(F.col("names").isNotNull(), check_required(F.col("names.primary"))), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_string_min_length_check() -> Check: + return Check( + field="names.primary", + name="string_min_length", + expr=check_string_min_length(F.col("names.primary"), 1), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_primary_stripped_check() -> Check: + return Check( + field="names.primary", + name="stripped", + expr=check_stripped(F.col("names.primary")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"names"}), + ) + + +def _names_common_key_check() -> Check: + return Check( + field="names.common{key}", + name="language_tag", + expr=map_keys_check( + "names.common", + lambda k: check_pattern( + k, + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_common_value_check() -> Check: + return Check( + field="names.common{value}", + name="stripped", + expr=map_values_check("names.common", lambda v: check_stripped(v)), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_required_check() -> Check: + return Check( + field="names.rules[].value", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_string_min_length_check() -> Check: + return Check( + field="names.rules[].value", + name="string_min_length", + expr=array_check( + "names.rules", lambda el: check_string_min_length(el["value"], 1) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_value_stripped_check() -> Check: + return Check( + field="names.rules[].value", + name="stripped", + expr=array_check("names.rules", lambda el: check_stripped(el["value"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_required_check() -> Check: + return Check( + field="names.rules[].variant", + name="required", + expr=array_check("names.rules", lambda el: check_required(el["variant"])), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_variant_enum_check() -> Check: + return Check( + field="names.rules[].variant", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["variant"], ["common", "official", "alternate", "short"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_language_check() -> Check: + return Check( + field="names.rules[].language", + name="language_tag", + expr=array_check( + "names.rules", + lambda el: check_pattern( + el["language"], + "^(?:(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}?)|(?:[A-Za-z]{4,8}))(?:-[A-Za-z]{4})?(?:-[A-Za-z]{2}|[0-9]{3})?(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(?:-[A-WY-Za-wy-z0-9](?:-[A-Za-z0-9]{2,8})+)*\\z", + label="IETF BCP-47 language tag", + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_required_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["mode"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_mode_enum_check() -> Check: + return Check( + field="names.rules[].perspectives.mode", + name="enum", + expr=array_check( + "names.rules", + lambda el: check_enum( + el["perspectives"]["mode"], ["accepted_by", "disputed_by"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check() -> Check: + return Check( + field="names.rules[].perspectives.countries", + name="required", + expr=array_check( + "names.rules", + lambda el: F.when( + el["perspectives"].isNotNull(), + check_required(el["perspectives"]["countries"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_min_length_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_min_length", + name="array_min_length", + expr=array_check( + "names.rules", + lambda el: check_array_min_length(el["perspectives"]["countries"], 1), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_unique_check() -> Check: + return Check( + field="names.rules[].perspectives.countries_unique", + name="struct_unique", + expr=array_check( + "names.rules", + lambda el: check_struct_unique(el["perspectives"]["countries"]), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_perspectives_countries_check_1() -> Check: + return Check( + field="names.rules[].perspectives.countries[]", + name="country_code_alpha2", + expr=nested_array_check( + "names.rules", + lambda el: array_check( + el["perspectives"]["countries"], + lambda inner: check_pattern( + inner, "^[A-Z]{2}\\z", label="ISO 3166-1 alpha-2 country code" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_length_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_length", + expr=array_check( + "names.rules", lambda el: check_linear_range_length(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_bounds", + expr=array_check( + "names.rules", lambda el: check_linear_range_bounds(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_between_linear_range_order_check() -> Check: + return Check( + field="names.rules[].between", + name="linear_range_order", + expr=array_check( + "names.rules", lambda el: check_linear_range_order(el["between"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _names_rules_side_check() -> Check: + return Check( + field="names.rules[].side", + name="enum", + expr=array_check( + "names.rules", lambda el: check_enum(el["side"], ["left", "right"]) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"names"}), + ) + + +def _class_required_check() -> Check: + return Check( + field="class_0", + name="required", + expr=F.when(F.col("subtype").isin(["road"]), check_required(F.col("class"))), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class", "subtype"}), + ) + + +def _class_enum_check() -> Check: + return Check( + field="class_0", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + check_enum( + F.col("class"), + [ + "motorway", + "primary", + "secondary", + "tertiary", + "residential", + "living_street", + "trunk", + "unclassified", + "service", + "pedestrian", + "footway", + "steps", + "path", + "track", + "cycleway", + "bridleway", + "unknown", + ], + ), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class", "subtype"}), + ) + + +def _destinations_from_connector_id_required_check() -> Check: + return Check( + field="destinations[].from_connector_id", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "destinations", lambda el: check_required(el["from_connector_id"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_from_connector_id_string_min_length_check() -> Check: + return Check( + field="destinations[].from_connector_id", + name="string_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "destinations", + lambda el: check_string_min_length(el["from_connector_id"], 1), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_from_connector_id_no_whitespace_check() -> Check: + return Check( + field="destinations[].from_connector_id", + name="no_whitespace", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "destinations", + lambda el: check_pattern( + el["from_connector_id"], + "^\\S+\\z", + label="String without whitespace characters", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_to_connector_id_required_check() -> Check: + return Check( + field="destinations[].to_connector_id", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "destinations", lambda el: check_required(el["to_connector_id"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_to_connector_id_string_min_length_check() -> Check: + return Check( + field="destinations[].to_connector_id", + name="string_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "destinations", + lambda el: check_string_min_length(el["to_connector_id"], 1), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_to_connector_id_no_whitespace_check() -> Check: + return Check( + field="destinations[].to_connector_id", + name="no_whitespace", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "destinations", + lambda el: check_pattern( + el["to_connector_id"], + "^\\S+\\z", + label="String without whitespace characters", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_to_segment_id_required_check() -> Check: + return Check( + field="destinations[].to_segment_id", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check("destinations", lambda el: check_required(el["to_segment_id"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_to_segment_id_string_min_length_check() -> Check: + return Check( + field="destinations[].to_segment_id", + name="string_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "destinations", + lambda el: check_string_min_length(el["to_segment_id"], 1), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_to_segment_id_no_whitespace_check() -> Check: + return Check( + field="destinations[].to_segment_id", + name="no_whitespace", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "destinations", + lambda el: check_pattern( + el["to_segment_id"], + "^\\S+\\z", + label="String without whitespace characters", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_final_heading_required_check() -> Check: + return Check( + field="destinations[].final_heading", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check("destinations", lambda el: check_required(el["final_heading"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_final_heading_enum_check() -> Check: + return Check( + field="destinations[].final_heading", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "destinations", + lambda el: check_enum(el["final_heading"], ["forward", "backward"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_labels_min_length_check() -> Check: + return Check( + field="destinations[].labels_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "destinations", lambda el: check_array_min_length(el["labels"], 1) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_labels_unique_check() -> Check: + return Check( + field="destinations[].labels_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check("destinations", lambda el: check_struct_unique(el["labels"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_labels_value_required_check() -> Check: + return Check( + field="destinations[].labels[].value", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "destinations", + lambda el: array_check( + el["labels"], lambda inner: check_required(inner["value"]) + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_labels_value_string_min_length_check() -> Check: + return Check( + field="destinations[].labels[].value", + name="string_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "destinations", + lambda el: array_check( + el["labels"], + lambda inner: check_string_min_length(inner["value"], 1), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_labels_value_stripped_check() -> Check: + return Check( + field="destinations[].labels[].value", + name="stripped", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "destinations", + lambda el: array_check( + el["labels"], lambda inner: check_stripped(inner["value"]) + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_labels_type_required_check() -> Check: + return Check( + field="destinations[].labels[].type", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "destinations", + lambda el: array_check( + el["labels"], lambda inner: check_required(inner["type"]) + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_labels_type_enum_check() -> Check: + return Check( + field="destinations[].labels[].type", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "destinations", + lambda el: array_check( + el["labels"], + lambda inner: check_enum( + inner["type"], + [ + "street", + "country", + "route_ref", + "toward_route_ref", + "unknown", + ], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_symbols_unique_check() -> Check: + return Check( + field="destinations[].symbols_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check("destinations", lambda el: check_struct_unique(el["symbols"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_symbols_check() -> Check: + return Check( + field="destinations[].symbols[]", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "destinations", + lambda el: array_check( + el["symbols"], + lambda inner: check_enum( + inner, + [ + "motorway", + "airport", + "hospital", + "center", + "industrial", + "parking", + "bus", + "train_station", + "rest_area", + "ferry", + "motorroad", + "fuel", + "viewpoint", + "fuel_diesel", + "food", + "lodging", + "info", + "camp_site", + "interchange", + "restrooms", + ], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_when_heading_required_check() -> Check: + return Check( + field="destinations[].when.heading", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "destinations", + lambda el: F.when( + el["when"].isNotNull(), check_required(el["when"]["heading"]) + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _destinations_when_heading_enum_check() -> Check: + return Check( + field="destinations[].when.heading", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "destinations", + lambda el: check_enum(el["when"]["heading"], ["forward", "backward"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _prohibited_transitions_sequence_check() -> Check: + return Check( + field="prohibited_transitions[].sequence", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", lambda el: check_required(el["sequence"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_sequence_min_length_check() -> Check: + return Check( + field="prohibited_transitions[].sequence_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_array_min_length(el["sequence"], 1), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_sequence_unique_check() -> Check: + return Check( + field="prohibited_transitions[].sequence_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", lambda el: check_struct_unique(el["sequence"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_sequence_connector_id_required_check() -> Check: + return Check( + field="prohibited_transitions[].sequence[].connector_id", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["sequence"], lambda inner: check_required(inner["connector_id"]) + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_sequence_connector_id_string_min_length_check() -> Check: + return Check( + field="prohibited_transitions[].sequence[].connector_id", + name="string_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["sequence"], + lambda inner: check_string_min_length(inner["connector_id"], 1), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_sequence_connector_id_no_whitespace_check() -> Check: + return Check( + field="prohibited_transitions[].sequence[].connector_id", + name="no_whitespace", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["sequence"], + lambda inner: check_pattern( + inner["connector_id"], + "^\\S+\\z", + label="String without whitespace characters", + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_sequence_segment_id_required_check() -> Check: + return Check( + field="prohibited_transitions[].sequence[].segment_id", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["sequence"], lambda inner: check_required(inner["segment_id"]) + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_sequence_segment_id_string_min_length_check() -> Check: + return Check( + field="prohibited_transitions[].sequence[].segment_id", + name="string_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["sequence"], + lambda inner: check_string_min_length(inner["segment_id"], 1), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_sequence_segment_id_no_whitespace_check() -> Check: + return Check( + field="prohibited_transitions[].sequence[].segment_id", + name="no_whitespace", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["sequence"], + lambda inner: check_pattern( + inner["segment_id"], + "^\\S+\\z", + label="String without whitespace characters", + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_final_heading_required_check() -> Check: + return Check( + field="prohibited_transitions[].final_heading", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", lambda el: check_required(el["final_heading"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_final_heading_enum_check() -> Check: + return Check( + field="prohibited_transitions[].final_heading", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_enum(el["final_heading"], ["forward", "backward"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_between_linear_range_length_check() -> Check: + return Check( + field="prohibited_transitions[].between", + name="linear_range_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_linear_range_length(el["between"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_between_linear_range_bounds_check() -> Check: + return Check( + field="prohibited_transitions[].between", + name="linear_range_bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_linear_range_bounds(el["between"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_between_linear_range_order_check() -> Check: + return Check( + field="prohibited_transitions[].between", + name="linear_range_order", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_linear_range_order(el["between"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_heading_check() -> Check: + return Check( + field="prohibited_transitions[].when.heading", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_enum(el["when"]["heading"], ["forward", "backward"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_mode_min_length_check() -> Check: + return Check( + field="prohibited_transitions[].when.mode_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_array_min_length(el["when"]["mode"], 1), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_mode_unique_check() -> Check: + return Check( + field="prohibited_transitions[].when.mode_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_struct_unique(el["when"]["mode"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_mode_check() -> Check: + return Check( + field="prohibited_transitions[].when.mode[]", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["mode"], + lambda inner: check_enum( + inner, + [ + "vehicle", + "motor_vehicle", + "car", + "truck", + "motorcycle", + "foot", + "bicycle", + "bus", + "hgv", + "hov", + "emergency", + ], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_using_min_length_check() -> Check: + return Check( + field="prohibited_transitions[].when.using_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_array_min_length(el["when"]["using"], 1), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_using_unique_check() -> Check: + return Check( + field="prohibited_transitions[].when.using_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_struct_unique(el["when"]["using"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_using_check() -> Check: + return Check( + field="prohibited_transitions[].when.using[]", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["using"], + lambda inner: check_enum( + inner, + [ + "as_customer", + "at_destination", + "to_deliver", + "to_farm", + "for_forestry", + ], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_recognized_min_length_check() -> Check: + return Check( + field="prohibited_transitions[].when.recognized_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_array_min_length(el["when"]["recognized"], 1), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_recognized_unique_check() -> Check: + return Check( + field="prohibited_transitions[].when.recognized_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_struct_unique(el["when"]["recognized"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_recognized_check() -> Check: + return Check( + field="prohibited_transitions[].when.recognized[]", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["recognized"], + lambda inner: check_enum( + inner, + [ + "as_permitted", + "as_private", + "as_disabled", + "as_employee", + "as_student", + ], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_min_length_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_array_min_length(el["when"]["vehicle"], 1), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_unique_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "prohibited_transitions", + lambda el: check_struct_unique(el["when"]["vehicle"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_dimension_required_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].dimension", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_required(inner["dimension"]), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_dimension_enum_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].dimension", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_enum( + inner["dimension"], + ["axle_count", "height", "length", "weight", "width"], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_comparison_required_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].comparison", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_required(inner["comparison"]), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_comparison_enum_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].comparison", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_enum( + inner["comparison"], + [ + "greater_than", + "greater_than_equal", + "equal", + "less_than", + "less_than_equal", + ], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_value_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].value_0", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["axle_count"]), + check_required(inner["value"]), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_value_required_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].value_1", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin( + ["height", "length", "weight", "width"] + ), + check_required(inner["value"]), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_value_bounds_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].value", + name="bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin( + ["height", "length", "weight", "width"] + ), + check_bounds(inner["value"], ge=0.0), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_unit_required_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].unit_0", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["height", "length", "width"]), + check_required(inner["unit"]), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_unit_enum_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].unit_0", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["height", "length", "width"]), + check_enum( + inner["unit"], ["in", "ft", "yd", "mi", "cm", "m", "km"] + ), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_unit_required_check_1() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].unit_1", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["weight"]), + check_required(inner["unit"]), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_unit_enum_check_1() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].unit_1", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["weight"]), + check_enum( + inner["unit"], ["oz", "lb", "st", "lt", "g", "kg", "t"] + ), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _road_flags_min_length_check() -> Check: + return Check( + field="road_flags_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + check_array_min_length(F.col("road_flags"), 1), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"road_flags", "subtype"}), + ) + + +def _road_flags_unique_check() -> Check: + return Check( + field="road_flags_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), check_struct_unique(F.col("road_flags")) + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"road_flags", "subtype"}), + ) + + +def _road_flags_values_check() -> Check: + return Check( + field="road_flags[].values", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check("road_flags", lambda el: check_required(el["values"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"road_flags", "subtype"}), + ) + + +def _road_flags_values_min_length_check() -> Check: + return Check( + field="road_flags[].values_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "road_flags", lambda el: check_array_min_length(el["values"], 1) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"road_flags", "subtype"}), + ) + + +def _road_flags_values_unique_check() -> Check: + return Check( + field="road_flags[].values_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check("road_flags", lambda el: check_struct_unique(el["values"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"road_flags", "subtype"}), + ) + + +def _road_flags_values_check_1() -> Check: + return Check( + field="road_flags[].values[]", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "road_flags", + lambda el: array_check( + el["values"], + lambda inner: check_enum( + inner, + [ + "is_bridge", + "is_link", + "is_tunnel", + "is_under_construction", + "is_abandoned", + "is_covered", + "is_indoor", + ], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"road_flags", "subtype"}), + ) + + +def _road_flags_between_linear_range_length_check() -> Check: + return Check( + field="road_flags[].between", + name="linear_range_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "road_flags", lambda el: check_linear_range_length(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"road_flags", "subtype"}), + ) + + +def _road_flags_between_linear_range_bounds_check() -> Check: + return Check( + field="road_flags[].between", + name="linear_range_bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "road_flags", lambda el: check_linear_range_bounds(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"road_flags", "subtype"}), + ) + + +def _road_flags_between_linear_range_order_check() -> Check: + return Check( + field="road_flags[].between", + name="linear_range_order", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "road_flags", lambda el: check_linear_range_order(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"road_flags", "subtype"}), + ) + + +def _road_surface_min_length_check() -> Check: + return Check( + field="road_surface_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + check_array_min_length(F.col("road_surface"), 1), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"road_surface", "subtype"}), + ) + + +def _road_surface_unique_check() -> Check: + return Check( + field="road_surface_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), check_struct_unique(F.col("road_surface")) + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"road_surface", "subtype"}), + ) + + +def _road_surface_value_required_check() -> Check: + return Check( + field="road_surface[].value", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check("road_surface", lambda el: check_required(el["value"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"road_surface", "subtype"}), + ) + + +def _road_surface_value_enum_check() -> Check: + return Check( + field="road_surface[].value", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "road_surface", + lambda el: check_enum( + el["value"], + [ + "unknown", + "paved", + "unpaved", + "gravel", + "dirt", + "paving_stones", + "metal", + ], + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"road_surface", "subtype"}), + ) + + +def _road_surface_between_linear_range_length_check() -> Check: + return Check( + field="road_surface[].between", + name="linear_range_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "road_surface", lambda el: check_linear_range_length(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"road_surface", "subtype"}), + ) + + +def _road_surface_between_linear_range_bounds_check() -> Check: + return Check( + field="road_surface[].between", + name="linear_range_bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "road_surface", lambda el: check_linear_range_bounds(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"road_surface", "subtype"}), + ) + + +def _road_surface_between_linear_range_order_check() -> Check: + return Check( + field="road_surface[].between", + name="linear_range_order", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "road_surface", lambda el: check_linear_range_order(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"road_surface", "subtype"}), + ) + + +def _speed_limits_min_length_check() -> Check: + return Check( + field="speed_limits_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + check_array_min_length(F.col("speed_limits"), 1), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_unique_check() -> Check: + return Check( + field="speed_limits_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), check_struct_unique(F.col("speed_limits")) + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_max_speed_value_required_check() -> Check: + return Check( + field="speed_limits[].max_speed.value", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: F.when( + el["max_speed"].isNotNull(), + check_required(el["max_speed"]["value"]), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_max_speed_value_bounds_check() -> Check: + return Check( + field="speed_limits[].max_speed.value_0", + name="bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: check_bounds( + el["max_speed"]["value"], ge=1, check_nan=False + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_max_speed_value_bounds_check_1() -> Check: + return Check( + field="speed_limits[].max_speed.value_1", + name="bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: check_bounds( + el["max_speed"]["value"], le=350, check_nan=False + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_max_speed_unit_required_check() -> Check: + return Check( + field="speed_limits[].max_speed.unit", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: F.when( + el["max_speed"].isNotNull(), check_required(el["max_speed"]["unit"]) + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_max_speed_unit_enum_check() -> Check: + return Check( + field="speed_limits[].max_speed.unit", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: check_enum(el["max_speed"]["unit"], ["mph", "km/h"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_min_speed_value_required_check() -> Check: + return Check( + field="speed_limits[].min_speed.value", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: F.when( + el["min_speed"].isNotNull(), + check_required(el["min_speed"]["value"]), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_min_speed_value_bounds_check() -> Check: + return Check( + field="speed_limits[].min_speed.value_0", + name="bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: check_bounds( + el["min_speed"]["value"], ge=1, check_nan=False + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_min_speed_value_bounds_check_1() -> Check: + return Check( + field="speed_limits[].min_speed.value_1", + name="bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: check_bounds( + el["min_speed"]["value"], le=350, check_nan=False + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_min_speed_unit_required_check() -> Check: + return Check( + field="speed_limits[].min_speed.unit", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: F.when( + el["min_speed"].isNotNull(), check_required(el["min_speed"]["unit"]) + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_min_speed_unit_enum_check() -> Check: + return Check( + field="speed_limits[].min_speed.unit", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: check_enum(el["min_speed"]["unit"], ["mph", "km/h"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_between_linear_range_length_check() -> Check: + return Check( + field="speed_limits[].between", + name="linear_range_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", lambda el: check_linear_range_length(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_between_linear_range_bounds_check() -> Check: + return Check( + field="speed_limits[].between", + name="linear_range_bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", lambda el: check_linear_range_bounds(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_between_linear_range_order_check() -> Check: + return Check( + field="speed_limits[].between", + name="linear_range_order", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", lambda el: check_linear_range_order(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_heading_check() -> Check: + return Check( + field="speed_limits[].when.heading", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: check_enum(el["when"]["heading"], ["forward", "backward"]), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_mode_min_length_check() -> Check: + return Check( + field="speed_limits[].when.mode_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", lambda el: check_array_min_length(el["when"]["mode"], 1) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_mode_unique_check() -> Check: + return Check( + field="speed_limits[].when.mode_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", lambda el: check_struct_unique(el["when"]["mode"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_mode_check() -> Check: + return Check( + field="speed_limits[].when.mode[]", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["mode"], + lambda inner: check_enum( + inner, + [ + "vehicle", + "motor_vehicle", + "car", + "truck", + "motorcycle", + "foot", + "bicycle", + "bus", + "hgv", + "hov", + "emergency", + ], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_using_min_length_check() -> Check: + return Check( + field="speed_limits[].when.using_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: check_array_min_length(el["when"]["using"], 1), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_using_unique_check() -> Check: + return Check( + field="speed_limits[].when.using_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", lambda el: check_struct_unique(el["when"]["using"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_using_check() -> Check: + return Check( + field="speed_limits[].when.using[]", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["using"], + lambda inner: check_enum( + inner, + [ + "as_customer", + "at_destination", + "to_deliver", + "to_farm", + "for_forestry", + ], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_recognized_min_length_check() -> Check: + return Check( + field="speed_limits[].when.recognized_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: check_array_min_length(el["when"]["recognized"], 1), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_recognized_unique_check() -> Check: + return Check( + field="speed_limits[].when.recognized_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", lambda el: check_struct_unique(el["when"]["recognized"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_recognized_check() -> Check: + return Check( + field="speed_limits[].when.recognized[]", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["recognized"], + lambda inner: check_enum( + inner, + [ + "as_permitted", + "as_private", + "as_disabled", + "as_employee", + "as_student", + ], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_min_length_check() -> Check: + return Check( + field="speed_limits[].when.vehicle_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", + lambda el: check_array_min_length(el["when"]["vehicle"], 1), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_unique_check() -> Check: + return Check( + field="speed_limits[].when.vehicle_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "speed_limits", lambda el: check_struct_unique(el["when"]["vehicle"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_dimension_required_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].dimension", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_required(inner["dimension"]), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_dimension_enum_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].dimension", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_enum( + inner["dimension"], + ["axle_count", "height", "length", "weight", "width"], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_comparison_required_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].comparison", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_required(inner["comparison"]), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_comparison_enum_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].comparison", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_enum( + inner["comparison"], + [ + "greater_than", + "greater_than_equal", + "equal", + "less_than", + "less_than_equal", + ], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_value_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].value_0", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["axle_count"]), + check_required(inner["value"]), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_value_required_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].value_1", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin( + ["height", "length", "weight", "width"] + ), + check_required(inner["value"]), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_value_bounds_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].value", + name="bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin( + ["height", "length", "weight", "width"] + ), + check_bounds(inner["value"], ge=0.0), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_unit_required_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].unit_0", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["height", "length", "width"]), + check_required(inner["unit"]), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_unit_enum_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].unit_0", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["height", "length", "width"]), + check_enum( + inner["unit"], ["in", "ft", "yd", "mi", "cm", "m", "km"] + ), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_unit_required_check_1() -> Check: + return Check( + field="speed_limits[].when.vehicle[].unit_1", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["weight"]), + check_required(inner["unit"]), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_unit_enum_check_1() -> Check: + return Check( + field="speed_limits[].when.vehicle[].unit_1", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["weight"]), + check_enum( + inner["unit"], ["oz", "lb", "st", "lt", "g", "kg", "t"] + ), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _subclass_check() -> Check: + return Check( + field="subclass", + name="enum", + expr=F.when( + F.col("subtype").isin(["road"]), + check_enum( + F.col("subclass"), + [ + "link", + "sidewalk", + "crosswalk", + "parking_aisle", + "driveway", + "alley", + "cycle_crossing", + ], + ), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subclass", "subtype"}), + ) + + +def _width_rules_min_length_check() -> Check: + return Check( + field="width_rules_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["road"]), + check_array_min_length(F.col("width_rules"), 1), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype", "width_rules"}), + ) + + +def _width_rules_unique_check() -> Check: + return Check( + field="width_rules_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["road"]), check_struct_unique(F.col("width_rules")) + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype", "width_rules"}), + ) + + +def _width_rules_value_required_check() -> Check: + return Check( + field="width_rules[].value", + name="required", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check("width_rules", lambda el: check_required(el["value"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"subtype", "width_rules"}), + ) + + +def _width_rules_value_bounds_check() -> Check: + return Check( + field="width_rules[].value", + name="bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check("width_rules", lambda el: check_bounds(el["value"], gt=0.0)), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"subtype", "width_rules"}), + ) + + +def _width_rules_between_linear_range_length_check() -> Check: + return Check( + field="width_rules[].between", + name="linear_range_length", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "width_rules", lambda el: check_linear_range_length(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"subtype", "width_rules"}), + ) + + +def _width_rules_between_linear_range_bounds_check() -> Check: + return Check( + field="width_rules[].between", + name="linear_range_bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "width_rules", lambda el: check_linear_range_bounds(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"subtype", "width_rules"}), + ) + + +def _width_rules_between_linear_range_order_check() -> Check: + return Check( + field="width_rules[].between", + name="linear_range_order", + expr=F.when( + F.col("subtype").isin(["road"]), + array_check( + "width_rules", lambda el: check_linear_range_order(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"subtype", "width_rules"}), + ) + + +def _class_required_check_1() -> Check: + return Check( + field="class_1", + name="required", + expr=F.when(F.col("subtype").isin(["rail"]), check_required(F.col("class"))), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class", "subtype"}), + ) + + +def _class_enum_check_1() -> Check: + return Check( + field="class_1", + name="enum", + expr=F.when( + F.col("subtype").isin(["rail"]), + check_enum( + F.col("class"), + [ + "funicular", + "light_rail", + "monorail", + "narrow_gauge", + "standard_gauge", + "subway", + "tram", + "unknown", + ], + ), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class", "subtype"}), + ) + + +def _rail_flags_min_length_check() -> Check: + return Check( + field="rail_flags_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["rail"]), + check_array_min_length(F.col("rail_flags"), 1), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"rail_flags", "subtype"}), + ) + + +def _rail_flags_unique_check() -> Check: + return Check( + field="rail_flags_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["rail"]), check_struct_unique(F.col("rail_flags")) + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"rail_flags", "subtype"}), + ) + + +def _rail_flags_values_check() -> Check: + return Check( + field="rail_flags[].values", + name="required", + expr=F.when( + F.col("subtype").isin(["rail"]), + array_check("rail_flags", lambda el: check_required(el["values"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"rail_flags", "subtype"}), + ) + + +def _rail_flags_values_min_length_check() -> Check: + return Check( + field="rail_flags[].values_min_length", + name="array_min_length", + expr=F.when( + F.col("subtype").isin(["rail"]), + array_check( + "rail_flags", lambda el: check_array_min_length(el["values"], 1) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"rail_flags", "subtype"}), + ) + + +def _rail_flags_values_unique_check() -> Check: + return Check( + field="rail_flags[].values_unique", + name="struct_unique", + expr=F.when( + F.col("subtype").isin(["rail"]), + array_check("rail_flags", lambda el: check_struct_unique(el["values"])), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"rail_flags", "subtype"}), + ) + + +def _rail_flags_values_check_1() -> Check: + return Check( + field="rail_flags[].values[]", + name="enum", + expr=F.when( + F.col("subtype").isin(["rail"]), + nested_array_check( + "rail_flags", + lambda el: array_check( + el["values"], + lambda inner: check_enum( + inner, + [ + "is_bridge", + "is_tunnel", + "is_under_construction", + "is_abandoned", + "is_covered", + "is_passenger", + "is_freight", + "is_disused", + ], + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"rail_flags", "subtype"}), + ) + + +def _rail_flags_between_linear_range_length_check() -> Check: + return Check( + field="rail_flags[].between", + name="linear_range_length", + expr=F.when( + F.col("subtype").isin(["rail"]), + array_check( + "rail_flags", lambda el: check_linear_range_length(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"rail_flags", "subtype"}), + ) + + +def _rail_flags_between_linear_range_bounds_check() -> Check: + return Check( + field="rail_flags[].between", + name="linear_range_bounds", + expr=F.when( + F.col("subtype").isin(["rail"]), + array_check( + "rail_flags", lambda el: check_linear_range_bounds(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"rail_flags", "subtype"}), + ) + + +def _rail_flags_between_linear_range_order_check() -> Check: + return Check( + field="rail_flags[].between", + name="linear_range_order", + expr=F.when( + F.col("subtype").isin(["rail"]), + array_check( + "rail_flags", lambda el: check_linear_range_order(el["between"]) + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"rail_flags", "subtype"}), + ) + + +def _access_restrictions_when_vehicle_check_forbid_if_0_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].unit_forbidden", + name="forbid_if", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_forbid_if( + inner["unit"], + inner["dimension"] == "axle_count", + "dimension = 'axle_count'", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_check_require_if_1_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].unit_required_0", + name="require_if", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_require_if( + inner["unit"], + inner["dimension"] == "height", + "dimension = 'height'", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_check_require_if_2_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].unit_required_1", + name="require_if", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_require_if( + inner["unit"], + inner["dimension"] == "length", + "dimension = 'length'", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_check_require_if_3_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].unit_required_2", + name="require_if", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_require_if( + inner["unit"], + inner["dimension"] == "weight", + "dimension = 'weight'", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_check_require_if_4_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].unit_required_3", + name="require_if", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_require_if( + inner["unit"], inner["dimension"] == "width", "dimension = 'width'" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_check_require_any_of_5_check() -> Check: + return Check( + field="access_restrictions[].when", + name="require_any_of", + expr=array_check( + "access_restrictions", + lambda el: F.when( + el["when"].isNotNull(), + check_require_any_of( + [ + el["when"]["heading"], + el["when"]["during"], + el["when"]["mode"], + el["when"]["using"], + el["when"]["recognized"], + el["when"]["vehicle"], + ], + ["heading", "during", "mode", "using", "recognized", "vehicle"], + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _destinations_check_require_any_of_6_check() -> Check: + return Check( + field="destinations[]", + name="require_any_of", + expr=array_check( + "destinations", + lambda el: check_require_any_of( + [el["labels"], el["symbols"]], ["labels", "symbols"] + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"destinations"}), + ) + + +def _prohibited_transitions_when_vehicle_check_forbid_if_7_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].unit_forbidden", + name="forbid_if", + expr=nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_forbid_if( + inner["unit"], + inner["dimension"] == "axle_count", + "dimension = 'axle_count'", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions"}), + ) + + +def _prohibited_transitions_when_vehicle_check_require_if_8_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].unit_required_0", + name="require_if", + expr=nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_require_if( + inner["unit"], + inner["dimension"] == "height", + "dimension = 'height'", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions"}), + ) + + +def _prohibited_transitions_when_vehicle_check_require_if_9_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].unit_required_1", + name="require_if", + expr=nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_require_if( + inner["unit"], + inner["dimension"] == "length", + "dimension = 'length'", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions"}), + ) + + +def _prohibited_transitions_when_vehicle_check_require_if_10_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].unit_required_2", + name="require_if", + expr=nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_require_if( + inner["unit"], + inner["dimension"] == "weight", + "dimension = 'weight'", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions"}), + ) + + +def _prohibited_transitions_when_vehicle_check_require_if_11_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].unit_required_3", + name="require_if", + expr=nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_require_if( + inner["unit"], inner["dimension"] == "width", "dimension = 'width'" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions"}), + ) + + +def _prohibited_transitions_when_check_require_any_of_12_check() -> Check: + return Check( + field="prohibited_transitions[].when", + name="require_any_of", + expr=array_check( + "prohibited_transitions", + lambda el: F.when( + el["when"].isNotNull(), + check_require_any_of( + [ + el["when"]["heading"], + el["when"]["during"], + el["when"]["mode"], + el["when"]["using"], + el["when"]["recognized"], + el["when"]["vehicle"], + ], + ["heading", "during", "mode", "using", "recognized", "vehicle"], + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions"}), + ) + + +def _speed_limits_when_vehicle_check_forbid_if_13_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].unit_forbidden", + name="forbid_if", + expr=nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_forbid_if( + inner["unit"], + inner["dimension"] == "axle_count", + "dimension = 'axle_count'", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits"}), + ) + + +def _speed_limits_when_vehicle_check_require_if_14_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].unit_required_0", + name="require_if", + expr=nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_require_if( + inner["unit"], + inner["dimension"] == "height", + "dimension = 'height'", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits"}), + ) + + +def _speed_limits_when_vehicle_check_require_if_15_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].unit_required_1", + name="require_if", + expr=nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_require_if( + inner["unit"], + inner["dimension"] == "length", + "dimension = 'length'", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits"}), + ) + + +def _speed_limits_when_vehicle_check_require_if_16_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].unit_required_2", + name="require_if", + expr=nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_require_if( + inner["unit"], + inner["dimension"] == "weight", + "dimension = 'weight'", + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits"}), + ) + + +def _speed_limits_when_vehicle_check_require_if_17_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].unit_required_3", + name="require_if", + expr=nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: check_require_if( + inner["unit"], inner["dimension"] == "width", "dimension = 'width'" + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits"}), + ) + + +def _speed_limits_when_check_require_any_of_18_check() -> Check: + return Check( + field="speed_limits[].when", + name="require_any_of", + expr=array_check( + "speed_limits", + lambda el: F.when( + el["when"].isNotNull(), + check_require_any_of( + [ + el["when"]["heading"], + el["when"]["during"], + el["when"]["mode"], + el["when"]["using"], + el["when"]["recognized"], + el["when"]["vehicle"], + ], + ["heading", "during", "mode", "using", "recognized", "vehicle"], + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits"}), + ) + + +def _speed_limits_check_require_any_of_19_check() -> Check: + return Check( + field="speed_limits[]", + name="require_any_of", + expr=array_check( + "speed_limits", + lambda el: check_require_any_of( + [el["max_speed"]["value"], el["min_speed"]["value"]], + ["max_speed.value", "min_speed.value"], + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits"}), + ) + + +def _check_forbid_if_20_check() -> Check: + return Check( + field="class_forbidden", + name="forbid_if", + expr=check_forbid_if( + F.col("class"), F.col("subtype") == "water", "subtype = 'water'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class", "subtype"}), + ) + + +def _check_require_if_21_check() -> Check: + return Check( + field="class_required_0", + name="require_if", + expr=check_require_if( + F.col("class"), F.col("subtype") == "rail", "subtype = 'rail'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class", "subtype"}), + ) + + +def _check_require_if_22_check() -> Check: + return Check( + field="class_required_1", + name="require_if", + expr=check_require_if( + F.col("class"), F.col("subtype") == "road", "subtype = 'road'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"class", "subtype"}), + ) + + +def _check_forbid_if_23_check() -> Check: + return Check( + field="destinations_forbidden", + name="forbid_if", + expr=check_forbid_if( + F.col("destinations"), F.col("subtype") != "road", "subtype != 'road'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"destinations", "subtype"}), + ) + + +def _check_forbid_if_24_check() -> Check: + return Check( + field="prohibited_transitions_forbidden", + name="forbid_if", + expr=check_forbid_if( + F.col("prohibited_transitions"), + F.col("subtype") != "road", + "subtype != 'road'", + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _check_forbid_if_25_check() -> Check: + return Check( + field="road_flags_forbidden", + name="forbid_if", + expr=check_forbid_if( + F.col("road_flags"), F.col("subtype") != "road", "subtype != 'road'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"road_flags", "subtype"}), + ) + + +def _check_forbid_if_26_check() -> Check: + return Check( + field="road_surface_forbidden", + name="forbid_if", + expr=check_forbid_if( + F.col("road_surface"), F.col("subtype") != "road", "subtype != 'road'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"road_surface", "subtype"}), + ) + + +def _check_forbid_if_27_check() -> Check: + return Check( + field="speed_limits_forbidden", + name="forbid_if", + expr=check_forbid_if( + F.col("speed_limits"), F.col("subtype") != "road", "subtype != 'road'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _check_forbid_if_28_check() -> Check: + return Check( + field="subclass_forbidden", + name="forbid_if", + expr=check_forbid_if( + F.col("subclass"), F.col("subtype") != "road", "subtype != 'road'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subclass", "subtype"}), + ) + + +def _check_forbid_if_29_check() -> Check: + return Check( + field="width_rules_forbidden", + name="forbid_if", + expr=check_forbid_if( + F.col("width_rules"), F.col("subtype") != "road", "subtype != 'road'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype", "width_rules"}), + ) + + +def _check_forbid_if_30_check() -> Check: + return Check( + field="rail_flags_forbidden", + name="forbid_if", + expr=check_forbid_if( + F.col("rail_flags"), F.col("subtype") != "rail", "subtype != 'rail'" + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"rail_flags", "subtype"}), + ) + + +def segment_checks() -> list[Check]: + """All validation checks for segment.""" + return [ + _id_required_check(), + _id_string_min_length_check(), + _id_no_whitespace_check(), + _bbox_bbox_completeness_check(), + _bbox_bbox_lat_ordering_check(), + _bbox_bbox_lat_range_check(), + _geometry_required_check(), + _geometry_geometry_type_check(), + _theme_required_check(), + _theme_enum_check(), + _type_required_check(), + _type_enum_check(), + _version_required_check(), + _version_bounds_check(), + _sources_min_length_check(), + _sources_unique_check(), + _sources_property_required_check(), + _sources_property_json_pointer_check(), + _sources_dataset_check(), + _sources_license_check(), + _sources_confidence_bounds_check(), + _sources_confidence_bounds_check_1(), + _sources_provider_string_min_length_check(), + _sources_provider_snake_case_check(), + _sources_resource_string_min_length_check(), + _sources_resource_snake_case_check(), + _sources_version_string_min_length_check(), + _sources_version_no_whitespace_check(), + _sources_between_linear_range_length_check(), + _sources_between_linear_range_bounds_check(), + _sources_between_linear_range_order_check(), + _subtype_required_check(), + _subtype_enum_check(), + _access_restrictions_min_length_check(), + _access_restrictions_unique_check(), + _access_restrictions_access_type_required_check(), + _access_restrictions_access_type_enum_check(), + _access_restrictions_between_linear_range_length_check(), + _access_restrictions_between_linear_range_bounds_check(), + _access_restrictions_between_linear_range_order_check(), + _access_restrictions_when_heading_check(), + _access_restrictions_when_mode_min_length_check(), + _access_restrictions_when_mode_unique_check(), + _access_restrictions_when_mode_check(), + _access_restrictions_when_using_min_length_check(), + _access_restrictions_when_using_unique_check(), + _access_restrictions_when_using_check(), + _access_restrictions_when_recognized_min_length_check(), + _access_restrictions_when_recognized_unique_check(), + _access_restrictions_when_recognized_check(), + _access_restrictions_when_vehicle_min_length_check(), + _access_restrictions_when_vehicle_unique_check(), + _access_restrictions_when_vehicle_dimension_required_check(), + _access_restrictions_when_vehicle_dimension_enum_check(), + _access_restrictions_when_vehicle_comparison_required_check(), + _access_restrictions_when_vehicle_comparison_enum_check(), + _access_restrictions_when_vehicle_value_check(), + _access_restrictions_when_vehicle_value_required_check(), + _access_restrictions_when_vehicle_value_bounds_check(), + _access_restrictions_when_vehicle_unit_required_check(), + _access_restrictions_when_vehicle_unit_enum_check(), + _access_restrictions_when_vehicle_unit_required_check_1(), + _access_restrictions_when_vehicle_unit_enum_check_1(), + _connectors_min_length_check(), + _connectors_unique_check(), + _connectors_connector_id_required_check(), + _connectors_connector_id_string_min_length_check(), + _connectors_connector_id_no_whitespace_check(), + _connectors_at_bounds_check(), + _connectors_at_bounds_check_1(), + _level_rules_value_check(), + _level_rules_between_linear_range_length_check(), + _level_rules_between_linear_range_bounds_check(), + _level_rules_between_linear_range_order_check(), + _routes_name_string_min_length_check(), + _routes_name_stripped_check(), + _routes_network_string_min_length_check(), + _routes_network_stripped_check(), + _routes_ref_string_min_length_check(), + _routes_ref_stripped_check(), + _routes_symbol_string_min_length_check(), + _routes_symbol_stripped_check(), + _routes_wikidata_check(), + _routes_between_linear_range_length_check(), + _routes_between_linear_range_bounds_check(), + _routes_between_linear_range_order_check(), + _subclass_rules_value_required_check(), + _subclass_rules_value_enum_check(), + _subclass_rules_between_linear_range_length_check(), + _subclass_rules_between_linear_range_bounds_check(), + _subclass_rules_between_linear_range_order_check(), + _names_primary_required_check(), + _names_primary_string_min_length_check(), + _names_primary_stripped_check(), + _names_common_key_check(), + _names_common_value_check(), + _names_rules_value_required_check(), + _names_rules_value_string_min_length_check(), + _names_rules_value_stripped_check(), + _names_rules_variant_required_check(), + _names_rules_variant_enum_check(), + _names_rules_language_check(), + _names_rules_perspectives_mode_required_check(), + _names_rules_perspectives_mode_enum_check(), + _names_rules_perspectives_countries_check(), + _names_rules_perspectives_countries_min_length_check(), + _names_rules_perspectives_countries_unique_check(), + _names_rules_perspectives_countries_check_1(), + _names_rules_between_linear_range_length_check(), + _names_rules_between_linear_range_bounds_check(), + _names_rules_between_linear_range_order_check(), + _names_rules_side_check(), + _class_required_check(), + _class_enum_check(), + _destinations_from_connector_id_required_check(), + _destinations_from_connector_id_string_min_length_check(), + _destinations_from_connector_id_no_whitespace_check(), + _destinations_to_connector_id_required_check(), + _destinations_to_connector_id_string_min_length_check(), + _destinations_to_connector_id_no_whitespace_check(), + _destinations_to_segment_id_required_check(), + _destinations_to_segment_id_string_min_length_check(), + _destinations_to_segment_id_no_whitespace_check(), + _destinations_final_heading_required_check(), + _destinations_final_heading_enum_check(), + _destinations_labels_min_length_check(), + _destinations_labels_unique_check(), + _destinations_labels_value_required_check(), + _destinations_labels_value_string_min_length_check(), + _destinations_labels_value_stripped_check(), + _destinations_labels_type_required_check(), + _destinations_labels_type_enum_check(), + _destinations_symbols_unique_check(), + _destinations_symbols_check(), + _destinations_when_heading_required_check(), + _destinations_when_heading_enum_check(), + _prohibited_transitions_sequence_check(), + _prohibited_transitions_sequence_min_length_check(), + _prohibited_transitions_sequence_unique_check(), + _prohibited_transitions_sequence_connector_id_required_check(), + _prohibited_transitions_sequence_connector_id_string_min_length_check(), + _prohibited_transitions_sequence_connector_id_no_whitespace_check(), + _prohibited_transitions_sequence_segment_id_required_check(), + _prohibited_transitions_sequence_segment_id_string_min_length_check(), + _prohibited_transitions_sequence_segment_id_no_whitespace_check(), + _prohibited_transitions_final_heading_required_check(), + _prohibited_transitions_final_heading_enum_check(), + _prohibited_transitions_between_linear_range_length_check(), + _prohibited_transitions_between_linear_range_bounds_check(), + _prohibited_transitions_between_linear_range_order_check(), + _prohibited_transitions_when_heading_check(), + _prohibited_transitions_when_mode_min_length_check(), + _prohibited_transitions_when_mode_unique_check(), + _prohibited_transitions_when_mode_check(), + _prohibited_transitions_when_using_min_length_check(), + _prohibited_transitions_when_using_unique_check(), + _prohibited_transitions_when_using_check(), + _prohibited_transitions_when_recognized_min_length_check(), + _prohibited_transitions_when_recognized_unique_check(), + _prohibited_transitions_when_recognized_check(), + _prohibited_transitions_when_vehicle_min_length_check(), + _prohibited_transitions_when_vehicle_unique_check(), + _prohibited_transitions_when_vehicle_dimension_required_check(), + _prohibited_transitions_when_vehicle_dimension_enum_check(), + _prohibited_transitions_when_vehicle_comparison_required_check(), + _prohibited_transitions_when_vehicle_comparison_enum_check(), + _prohibited_transitions_when_vehicle_value_check(), + _prohibited_transitions_when_vehicle_value_required_check(), + _prohibited_transitions_when_vehicle_value_bounds_check(), + _prohibited_transitions_when_vehicle_unit_required_check(), + _prohibited_transitions_when_vehicle_unit_enum_check(), + _prohibited_transitions_when_vehicle_unit_required_check_1(), + _prohibited_transitions_when_vehicle_unit_enum_check_1(), + _road_flags_min_length_check(), + _road_flags_unique_check(), + _road_flags_values_check(), + _road_flags_values_min_length_check(), + _road_flags_values_unique_check(), + _road_flags_values_check_1(), + _road_flags_between_linear_range_length_check(), + _road_flags_between_linear_range_bounds_check(), + _road_flags_between_linear_range_order_check(), + _road_surface_min_length_check(), + _road_surface_unique_check(), + _road_surface_value_required_check(), + _road_surface_value_enum_check(), + _road_surface_between_linear_range_length_check(), + _road_surface_between_linear_range_bounds_check(), + _road_surface_between_linear_range_order_check(), + _speed_limits_min_length_check(), + _speed_limits_unique_check(), + _speed_limits_max_speed_value_required_check(), + _speed_limits_max_speed_value_bounds_check(), + _speed_limits_max_speed_value_bounds_check_1(), + _speed_limits_max_speed_unit_required_check(), + _speed_limits_max_speed_unit_enum_check(), + _speed_limits_min_speed_value_required_check(), + _speed_limits_min_speed_value_bounds_check(), + _speed_limits_min_speed_value_bounds_check_1(), + _speed_limits_min_speed_unit_required_check(), + _speed_limits_min_speed_unit_enum_check(), + _speed_limits_between_linear_range_length_check(), + _speed_limits_between_linear_range_bounds_check(), + _speed_limits_between_linear_range_order_check(), + _speed_limits_when_heading_check(), + _speed_limits_when_mode_min_length_check(), + _speed_limits_when_mode_unique_check(), + _speed_limits_when_mode_check(), + _speed_limits_when_using_min_length_check(), + _speed_limits_when_using_unique_check(), + _speed_limits_when_using_check(), + _speed_limits_when_recognized_min_length_check(), + _speed_limits_when_recognized_unique_check(), + _speed_limits_when_recognized_check(), + _speed_limits_when_vehicle_min_length_check(), + _speed_limits_when_vehicle_unique_check(), + _speed_limits_when_vehicle_dimension_required_check(), + _speed_limits_when_vehicle_dimension_enum_check(), + _speed_limits_when_vehicle_comparison_required_check(), + _speed_limits_when_vehicle_comparison_enum_check(), + _speed_limits_when_vehicle_value_check(), + _speed_limits_when_vehicle_value_required_check(), + _speed_limits_when_vehicle_value_bounds_check(), + _speed_limits_when_vehicle_unit_required_check(), + _speed_limits_when_vehicle_unit_enum_check(), + _speed_limits_when_vehicle_unit_required_check_1(), + _speed_limits_when_vehicle_unit_enum_check_1(), + _subclass_check(), + _width_rules_min_length_check(), + _width_rules_unique_check(), + _width_rules_value_required_check(), + _width_rules_value_bounds_check(), + _width_rules_between_linear_range_length_check(), + _width_rules_between_linear_range_bounds_check(), + _width_rules_between_linear_range_order_check(), + _class_required_check_1(), + _class_enum_check_1(), + _rail_flags_min_length_check(), + _rail_flags_unique_check(), + _rail_flags_values_check(), + _rail_flags_values_min_length_check(), + _rail_flags_values_unique_check(), + _rail_flags_values_check_1(), + _rail_flags_between_linear_range_length_check(), + _rail_flags_between_linear_range_bounds_check(), + _rail_flags_between_linear_range_order_check(), + _access_restrictions_when_vehicle_check_forbid_if_0_check(), + _access_restrictions_when_vehicle_check_require_if_1_check(), + _access_restrictions_when_vehicle_check_require_if_2_check(), + _access_restrictions_when_vehicle_check_require_if_3_check(), + _access_restrictions_when_vehicle_check_require_if_4_check(), + _access_restrictions_when_check_require_any_of_5_check(), + _destinations_check_require_any_of_6_check(), + _prohibited_transitions_when_vehicle_check_forbid_if_7_check(), + _prohibited_transitions_when_vehicle_check_require_if_8_check(), + _prohibited_transitions_when_vehicle_check_require_if_9_check(), + _prohibited_transitions_when_vehicle_check_require_if_10_check(), + _prohibited_transitions_when_vehicle_check_require_if_11_check(), + _prohibited_transitions_when_check_require_any_of_12_check(), + _speed_limits_when_vehicle_check_forbid_if_13_check(), + _speed_limits_when_vehicle_check_require_if_14_check(), + _speed_limits_when_vehicle_check_require_if_15_check(), + _speed_limits_when_vehicle_check_require_if_16_check(), + _speed_limits_when_vehicle_check_require_if_17_check(), + _speed_limits_when_check_require_any_of_18_check(), + _speed_limits_check_require_any_of_19_check(), + _check_forbid_if_20_check(), + _check_require_if_21_check(), + _check_require_if_22_check(), + _check_forbid_if_23_check(), + _check_forbid_if_24_check(), + _check_forbid_if_25_check(), + _check_forbid_if_26_check(), + _check_forbid_if_27_check(), + _check_forbid_if_28_check(), + _check_forbid_if_29_check(), + _check_forbid_if_30_check(), + ] + + +SEGMENT_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", BBOX_STRUCT, True), + StructField("geometry", BinaryType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("version", IntegerType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("property", StringType(), True), + StructField("dataset", StringType(), True), + StructField("license", StringType(), True), + StructField("record_id", StringType(), True), + StructField("update_time", StringType(), True), + StructField("confidence", DoubleType(), True), + StructField("provider", StringType(), True), + StructField("resource", StringType(), True), + StructField("version", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField("subtype", StringType(), True), + StructField( + "access_restrictions", + ArrayType( + StructType( + [ + StructField("access_type", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + StructField( + "when", + StructType( + [ + StructField("heading", StringType(), True), + StructField("during", StringType(), True), + StructField( + "mode", ArrayType(StringType(), True), True + ), + StructField( + "using", ArrayType(StringType(), True), True + ), + StructField( + "recognized", + ArrayType(StringType(), True), + True, + ), + StructField( + "vehicle", + ArrayType( + StructType( + [ + StructField( + "dimension", StringType(), True + ), + StructField( + "comparison", StringType(), True + ), + StructField( + "value", DoubleType(), True + ), + StructField( + "unit", StringType(), True + ), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), + True, + ), + StructField( + "connectors", + ArrayType( + StructType( + [ + StructField("connector_id", StringType(), True), + StructField("at", DoubleType(), True), + ] + ), + True, + ), + True, + ), + StructField( + "level_rules", + ArrayType( + StructType( + [ + StructField("value", IntegerType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField( + "routes", + ArrayType( + StructType( + [ + StructField("name", StringType(), True), + StructField("network", StringType(), True), + StructField("ref", StringType(), True), + StructField("symbol", StringType(), True), + StructField("wikidata", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField( + "subclass_rules", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField( + "names", + StructType( + [ + StructField("primary", StringType(), True), + StructField( + "common", MapType(StringType(), StringType(), True), True + ), + StructField( + "rules", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("variant", StringType(), True), + StructField("language", StringType(), True), + StructField( + "perspectives", + StructType( + [ + StructField("mode", StringType(), True), + StructField( + "countries", + ArrayType(StringType(), True), + True, + ), + ] + ), + True, + ), + StructField( + "between", ArrayType(DoubleType(), True), True + ), + StructField("side", StringType(), True), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + StructField("class", StringType(), True), + StructField( + "destinations", + ArrayType( + StructType( + [ + StructField("from_connector_id", StringType(), True), + StructField("to_connector_id", StringType(), True), + StructField("to_segment_id", StringType(), True), + StructField("final_heading", StringType(), True), + StructField( + "labels", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("type", StringType(), True), + ] + ), + True, + ), + True, + ), + StructField("symbols", ArrayType(StringType(), True), True), + StructField( + "when", + StructType([StructField("heading", StringType(), True)]), + True, + ), + ] + ), + True, + ), + True, + ), + StructField( + "prohibited_transitions", + ArrayType( + StructType( + [ + StructField( + "sequence", + ArrayType( + StructType( + [ + StructField("connector_id", StringType(), True), + StructField("segment_id", StringType(), True), + ] + ), + True, + ), + True, + ), + StructField("final_heading", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + StructField( + "when", + StructType( + [ + StructField("heading", StringType(), True), + StructField("during", StringType(), True), + StructField( + "mode", ArrayType(StringType(), True), True + ), + StructField( + "using", ArrayType(StringType(), True), True + ), + StructField( + "recognized", + ArrayType(StringType(), True), + True, + ), + StructField( + "vehicle", + ArrayType( + StructType( + [ + StructField( + "dimension", StringType(), True + ), + StructField( + "comparison", StringType(), True + ), + StructField( + "value", DoubleType(), True + ), + StructField( + "unit", StringType(), True + ), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), + True, + ), + StructField( + "road_flags", + ArrayType( + StructType( + [ + StructField("values", ArrayType(StringType(), True), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField( + "road_surface", + ArrayType( + StructType( + [ + StructField("value", StringType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField( + "speed_limits", + ArrayType( + StructType( + [ + StructField( + "max_speed", + StructType( + [ + StructField("value", IntegerType(), True), + StructField("unit", StringType(), True), + ] + ), + True, + ), + StructField( + "min_speed", + StructType( + [ + StructField("value", IntegerType(), True), + StructField("unit", StringType(), True), + ] + ), + True, + ), + StructField("is_max_speed_variable", BooleanType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + StructField( + "when", + StructType( + [ + StructField("heading", StringType(), True), + StructField("during", StringType(), True), + StructField( + "mode", ArrayType(StringType(), True), True + ), + StructField( + "using", ArrayType(StringType(), True), True + ), + StructField( + "recognized", + ArrayType(StringType(), True), + True, + ), + StructField( + "vehicle", + ArrayType( + StructType( + [ + StructField( + "dimension", StringType(), True + ), + StructField( + "comparison", StringType(), True + ), + StructField( + "value", DoubleType(), True + ), + StructField( + "unit", StringType(), True + ), + ] + ), + True, + ), + True, + ), + ] + ), + True, + ), + ] + ), + True, + ), + True, + ), + StructField("subclass", StringType(), True), + StructField( + "width_rules", + ArrayType( + StructType( + [ + StructField("value", DoubleType(), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + StructField( + "rail_flags", + ArrayType( + StructType( + [ + StructField("values", ArrayType(StringType(), True), True), + StructField("between", ArrayType(DoubleType(), True), True), + ] + ), + True, + ), + True, + ), + ] +) + +GEOMETRY_TYPES: tuple[GeometryType, ...] = (GeometryType.LINE_STRING,) + +ENTRY_POINT = "overture.schema.transportation:Segment" + +PARTITIONS: dict[str, str] = {"theme": "transportation"} + +MODEL_VALIDATION = ModelValidation( + schema=SEGMENT_SCHEMA, + checks=segment_checks, + geometry_types=GEOMETRY_TYPES, +) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/py.typed b/packages/overture-schema-pyspark/src/overture/schema/pyspark/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/schema_check.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/schema_check.py new file mode 100644 index 000000000..d4f76eb55 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/schema_check.py @@ -0,0 +1,124 @@ +"""Schema comparison for structural validation. + +Recursively diffs two `StructType` objects and reports mismatches +as a flat list with dot-notation paths. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +from pyspark.sql.types import ( + ArrayType, + DataType, + MapType, + StructType, +) + +# First struct (`.`), array (`[]`), or map (`{key}`/`{value}`) step marker +# in an encoded path; everything before it is the top-level column. +_STEP_MARKER = re.compile(r"[.\[{]") + + +@dataclass(frozen=True) +class SchemaMismatch: + """One structural difference between actual and expected schemas. + + Parameters + ---------- + path + Dot-notation path to the field (e.g. `"bbox.xmin"`). + actual + Actual type name, or `"missing"` if the field is absent. + expected + Expected type name, or `"missing"` if the field is unexpected. + """ + + path: str + actual: str + expected: str + + @property + def root(self) -> str: + """Top-level schema column this mismatch belongs to. + + Strips the struct/array/map step markers `_compare` embeds in + `path` (e.g. `sources[].confidence` -> `sources`), leaving the + column name that matches a `Check.read_columns` entry. + """ + return _STEP_MARKER.split(self.path, maxsplit=1)[0] + + +def _type_name(dt: DataType) -> str: + """Short display name for a DataType (e.g. `"StringType"`).""" + return type(dt).__name__ + + +def _compare( + actual: DataType, + expected: DataType, + prefix: str, + out: list[SchemaMismatch], +) -> None: + """Recursively compare two DataType trees.""" + if isinstance(expected, StructType) and isinstance(actual, StructType): + _compare_structs(actual, expected, prefix, out) + return + + if isinstance(expected, ArrayType) and isinstance(actual, ArrayType): + _compare(actual.elementType, expected.elementType, f"{prefix}[]", out) + return + + if isinstance(expected, MapType) and isinstance(actual, MapType): + _compare(actual.keyType, expected.keyType, f"{prefix}{{key}}", out) + _compare(actual.valueType, expected.valueType, f"{prefix}{{value}}", out) + return + + if type(actual) is not type(expected): + out.append(SchemaMismatch(prefix, _type_name(actual), _type_name(expected))) + + +def _compare_structs( + actual: StructType, + expected: StructType, + prefix: str, + out: list[SchemaMismatch], +) -> None: + """Compare two StructTypes field by field.""" + actual_fields = {f.name: f for f in actual.fields} + expected_fields = {f.name: f for f in expected.fields} + + # Ordered union: actual fields first, then any expected-only fields appended. + all_names = dict.fromkeys([*actual_fields, *expected_fields]) + for name in all_names: + path = f"{prefix}.{name}" if prefix else name + a = actual_fields.get(name) + e = expected_fields.get(name) + if a is None and e is not None: + out.append(SchemaMismatch(path, "missing", _type_name(e.dataType))) + elif e is None and a is not None: + out.append(SchemaMismatch(path, _type_name(a.dataType), "missing")) + elif a is not None and e is not None: + _compare(a.dataType, e.dataType, path, out) + + +def compare_schemas(actual: StructType, expected: StructType) -> list[SchemaMismatch]: + """Compare two Spark schemas and return all mismatches. + + Parameters + ---------- + actual + Schema inferred from the data (e.g. `df.schema`). + expected + Declared expected schema for the feature type. + + Returns + ------- + list[SchemaMismatch] + Empty when schemas match. Each mismatch identifies the + dot-notation path and what differs. + """ + out: list[SchemaMismatch] = [] + _compare_structs(actual, expected, "", out) + return out diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/validate.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/validate.py new file mode 100644 index 000000000..f6161b4b0 --- /dev/null +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/validate.py @@ -0,0 +1,426 @@ +"""Validation pipeline for registered models. + +`validate_model()` is the primary entry point: it looks up the +model type in the registry, compares schemas, filters checks, and +evaluates them in a single pass. Returns a `ValidationResult` +carrying the evaluated DataFrame and metadata. + +Lower-level helpers (`evaluate_checks`, `filter_errors`, +`explain_errors`) are available for consumers needing finer control. +""" + +from __future__ import annotations + +import re +from collections import Counter +from collections.abc import Iterable +from dataclasses import dataclass + +from pyspark.sql import DataFrame +from pyspark.sql import functions as F +from pyspark.sql.types import StringType, StructField, StructType + +from overture.schema.system.discovery import ( + entry_point_class_alias, + resolve_entry_point_key, +) + +from ._registry import REGISTRY +from .check import Check, CheckShape +from .expressions.column_patterns import coalesce_errors +from .schema_check import SchemaMismatch, compare_schemas + + +def model_keys() -> list[str]: + """Canonical entry-point keys registered in the validation registry.""" + return sorted(REGISTRY) + + +def model_names() -> list[str]: + """All names `validate_model` accepts. + + Includes canonical entry-point keys and the snake-case class-name + aliases the resolver recognizes (only when an alias is unambiguous). + """ + aliases = { + name + for name, count in Counter(entry_point_class_alias(k) for k in REGISTRY).items() + if count == 1 + } + return sorted(set(REGISTRY) | aliases) + + +def _normalize_suppress( + suppress: Iterable[str | tuple[str, str] | Check], +) -> tuple[set[str], set[tuple[str, str]]]: + """Partition suppress entries into root field names and (field, name) pairs. + + Parameters + ---------- + suppress + Mix of bare field name strings, `(field, name)` tuples, and + `Check` objects. + + Returns + ------- + tuple[set[str], set[tuple[str, str]]] + `(root_fields, pairs)` where `root_fields` is bare field names + and `pairs` is `(field, name)` pairs extracted from tuples and + Check objects. + """ + root_fields: set[str] = set() + pairs: set[tuple[str, str]] = set() + for entry in suppress: + if isinstance(entry, str): + root_fields.add(entry) + elif isinstance(entry, Check): + pairs.add((entry.field, entry.name)) + else: + pairs.add(entry) + return root_fields, pairs + + +# Matches the `_err_` columns `evaluate_checks` appends; ordinary +# user columns starting with `_err_` (but not followed by digits only) +# are preserved. +_ERR_COLUMN = re.compile(r"^_err_\d+$") + +# Working/output columns `explain_errors` introduces beyond `_err_`: +# `_idx`/`_errors` are UNPIVOT scratch, `field`/`check`/`message` are its +# output contract. An input column sharing any of these names produces +# duplicate attributes -> AMBIGUOUS_REFERENCE. +_EXPLAIN_RESERVED = ("_idx", "_errors", "field", "check", "message") + + +def _non_error_columns(evaluated: DataFrame) -> list[str]: + """Column names excluding `_err_N` error columns appended by `evaluate_checks`.""" + return [c for c in evaluated.columns if not _ERR_COLUMN.match(c)] + + +def _reject_reserved_collisions(collisions: Iterable[str], reserved_label: str) -> None: + """Raise if any input column collides with a reserved working/output name. + + Parameters + ---------- + collisions + Input column names that collide with the reserved set. + reserved_label + Human-readable description of the reserved names, completing the + sentence `... collide with {reserved_label}`. + + Raises + ------ + ValueError + If `collisions` is non-empty. The message names the offending + columns and the remediation (rename or drop them). + """ + names = sorted(collisions) + if names: + raise ValueError( + f"input columns {names} collide with {reserved_label}; " + f"rename or drop them before validating" + ) + + +def evaluate_checks(df: DataFrame, checks: list[Check]) -> DataFrame: + """Append `_err_N` columns for each check. + + Returns the input DataFrame with one `array` column per check, + containing error messages (non-empty) or null/empty (no error). + + Raises + ------ + ValueError + If `df` already contains a `_err_` column. Appending the + working columns would shadow it (duplicate attributes), so the + collision is rejected -- most realistically a persisted + `result.evaluated` fed back through validation. + """ + _reject_reserved_collisions( + (c for c in df.columns if _ERR_COLUMN.match(c)), + "the reserved '_err_' columns evaluate_checks appends", + ) + error_cols = [] + for i, chk in enumerate(checks): + if chk.shape == CheckShape.SCALAR: + col = F.array_compact(F.array(chk.expr)) + else: + col = coalesce_errors(F.filter(chk.expr, lambda x: x.isNotNull())) + error_cols.append(col.cast("array").alias(f"_err_{i}")) + return df.select("*", *error_cols) + + +def _max_error_size(n: int) -> F.Column: + """Build a Column for the largest `_err_N` array size across all checks. + + Use `greatest()` instead of chaining OR across all checks. A 255-check + OR tree triggers Spark's CommutativeExpression.orderCommutative during + plan canonicalization, which is O(n²+) and OOMs the driver. `greatest()` + is not a CommutativeExpression, so the optimizer skips that path. + + Caller must guarantee `n >= 1`. + """ + err_sizes = [F.coalesce(F.size(F.col(f"_err_{i}")), F.lit(0)) for i in range(n)] + return err_sizes[0] if n == 1 else F.greatest(*err_sizes) + + +def filter_errors(evaluated: DataFrame, checks: list[Check]) -> DataFrame: + """Filter an evaluated DataFrame to rows with at least one error. + + Parameters + ---------- + evaluated + DataFrame produced by `evaluate_checks()`. + checks + Same check list passed to `evaluate_checks()`. + + Returns + ------- + DataFrame + Original columns only (`_err_N` columns stripped). + """ + return evaluated.filter(_max_error_size(len(checks)) > 0).select( + *_non_error_columns(evaluated) + ) + + +def explain_errors(evaluated: DataFrame, checks: list[Check]) -> DataFrame: + """Unpivot evaluated error columns into one row per violation. + + Parameters + ---------- + evaluated + DataFrame produced by `evaluate_checks()`. + checks + Same check list passed to `evaluate_checks()`. + + Returns + ------- + DataFrame + Schema: `, field, check, message`. + + Raises + ------ + ValueError + If an original column collides with a working/output name + (`_idx`, `_errors`, `field`, `check`, `message`). + """ + orig_cols = _non_error_columns(evaluated) + _reject_reserved_collisions( + (c for c in orig_cols if c in _EXPLAIN_RESERVED), + f"explain_errors' working/output columns {list(_EXPLAIN_RESERVED)}", + ) + n = len(checks) + if n == 0: + empty_schema = StructType( + [ + *evaluated.select(*orig_cols).schema.fields, + StructField("field", StringType(), True), + StructField("check", StringType(), True), + StructField("message", StringType(), True), + ] + ) + return evaluated.sparkSession.createDataFrame([], empty_schema) + stack_args = ", ".join(f"{i}, `_err_{i}`" for i in range(n)) + unpivoted = evaluated.select( + *orig_cols, + F.expr(f"stack({n}, {stack_args}) as (_idx, _errors)"), + ).filter(F.col("_errors").isNotNull() & (F.size("_errors") > 0)) + + exploded = unpivoted.select( + *orig_cols, + "_idx", + F.explode("_errors").alias("message"), + ) + + meta_df = evaluated.sparkSession.createDataFrame( + [(i, c.field, c.name) for i, c in enumerate(checks)], + ["_idx", "field", "check"], + ) + + return exploded.join(F.broadcast(meta_df), "_idx").select( + *orig_cols, "field", "check", "message" + ) + + +@dataclass(frozen=True) +class ValidationResult: + """Result of validate_model(). + + Consumer owns caching of `evaluated`. Call `error_rows()` for + the filtered view; use `explain_errors(result.evaluated, + result.checks)` for the opt-in UNPIVOT. + """ + + evaluated: DataFrame + checks: list[Check] + schema_mismatches: list[SchemaMismatch] + suppressed_checks: list[Check] + absent_columns: tuple[str, ...] = () + """Root columns present in the schema but absent from the data and not already skipped. + + Ordered by first appearance in `schema_mismatches`, deduplicated. + Matches the set of root fields whose checks `validate_model` silently + drops; callers use this to suggest `--skip-columns` without re-deriving + it from `schema_mismatches`. + """ + + def error_rows(self) -> DataFrame: + """Rows with at least one violation. Original columns only.""" + if not self.checks: + return self.evaluated.limit(0) + return filter_errors(self.evaluated, self.checks) + + def row_counts(self) -> tuple[int, int]: + """Count total and error rows in a single pass. + + Computes both counts with one aggregation over the evaluated + DataFrame, avoiding the need to cache before counting. + + Returns + ------- + tuple[int, int] + `(total_rows, error_rows)`. + """ + if not self.checks: + return self.evaluated.count(), 0 + max_err = _max_error_size(len(self.checks)) + row = self.evaluated.agg( + F.count(F.lit(1)).alias("total"), + F.coalesce(F.sum(F.when(max_err > 0, 1).otherwise(0)), F.lit(0)).alias( + "errors" + ), + ).first() + assert row is not None # aggregation on a DataFrame always produces a row + return row["total"], row["errors"] + + +def validate_model( + df: DataFrame, + model_type: str, + *, + skip_columns: Iterable[str] = (), + ignore_extra_columns: Iterable[str] = (), + suppress: Iterable[str | tuple[str, str] | Check] = (), +) -> ValidationResult: + """Validate a DataFrame against a registered model type. + + Parameters + ---------- + df + Input DataFrame to validate. + model_type + Registered model type name (e.g. `"building"`). + skip_columns + Columns declared absent from the data. Raises `ValueError` + if any are present in `df.columns`. + ignore_extra_columns + Columns that may be present in the data but absent from the + expected schema. + suppress + Checks to remove before evaluation. Bare strings suppress by + root field; tuples by exact `(field, name)`; Check objects + by extracting `(field, name)`. Raises `ValueError` if any + entry doesn't match a registered check. + + Raises + ------ + ValueError + If `model_type` isn't registered. Message includes the + sorted list of known types. + """ + model_type = resolve_entry_point_key(model_type, REGISTRY) + validation = REGISTRY[model_type] + skip = frozenset(skip_columns) + ignore_extra = frozenset(ignore_extra_columns) + suppress_roots, suppress_pairs = _normalize_suppress(suppress) + + # Validate skip_columns are actually absent + present = skip & set(df.columns) + if present: + raise ValueError( + f"skip_columns {sorted(present)} are present in the " + f"DataFrame; remove them from skip_columns or drop them " + f"from the data" + ) + + # Schema comparison with filtering + raw_mismatches = compare_schemas(df.schema, validation.schema) + mismatches = [] + for m in raw_mismatches: + if m.root in skip: + continue + if m.expected == "missing" and m.root in ignore_extra: + continue + mismatches.append(m) + + # Validate suppress entries match real checks before filtering. A bare + # suppress string names a column; it is valid when some check reads it, + # which is exactly when suppressing it would drop a check -- the same + # `read_columns` set that drives exclusion below. + all_checks = validation.checks() + valid_roots = {col for c in all_checks for col in c.read_columns} + valid_pairs = {(c.field, c.name) for c in all_checks} + unmatched_roots = suppress_roots - valid_roots + unmatched_pairs = suppress_pairs - valid_pairs + if unmatched_roots or unmatched_pairs: + parts = [] + if unmatched_roots: + parts.append(f"unknown root fields {sorted(unmatched_roots)}") + if unmatched_pairs: + parts.append(f"unknown (field, name) pairs {sorted(unmatched_pairs)}") + raise ValueError( + f"suppress entries don't match any check for {model_type!r}: " + + "; ".join(parts) + ) + + # Schema columns the data lacks. A check referencing an absent column + # raises an AnalysisException during Spark plan analysis, so such checks + # are dropped before evaluation via the `excluded` filter below -- the + # same filter skip_columns feeds. Only that check-filtering is shared: + # a skip_columns mismatch was suppressed by the loop above, so the + # caller sees no mismatch and validation continues; an absent-column + # mismatch stays in `mismatches` and is reported, so the caller (the + # CLI) aborts unless --skip-schema-check. `--skip-columns` opts into + # that suppression -- it is not a restatement of the default. + # Exclusion is column-granular, so filtering is all-or-nothing: if the + # data has the `bbox` struct but is missing only `bbox.xmin`, every check + # rooted at `bbox` is dropped, including checks on sub-fields that are + # present. Finer granularity would require sub-column awareness in Check, + # which it deliberately lacks. `m.root` strips the array/map step markers + # (`sources[].confidence` -> `sources`) so a nested absence still resolves + # to the top-level column. + absent_columns = tuple( + dict.fromkeys(m.root for m in mismatches if m.actual == "missing") + ) + + # Check filtering. A check is dropped when any column it reads is gone -- + # whether skipped or structurally absent -- so an unresolvable `F.col()` + # never reaches Spark. Suppression by column name is the same predicate + # over a different set: suppressing a column is treating it as absent. + excluded = skip | set(absent_columns) + + def _is_excluded(chk: Check) -> bool: + return not excluded.isdisjoint(chk.read_columns) + + kept: list[Check] = [] + suppressed: list[Check] = [] + for chk in all_checks: + if _is_excluded(chk): + continue # structurally absent, not tracked in suppressed + if not suppress_roots.isdisjoint(chk.read_columns): + suppressed.append(chk) + continue + if (chk.field, chk.name) in suppress_pairs: + suppressed.append(chk) + continue + kept.append(chk) + + evaluated = evaluate_checks(df, kept) + return ValidationResult( + evaluated=evaluated, + checks=kept, + schema_mismatches=mismatches, + suppressed_checks=suppressed, + absent_columns=absent_columns, + ) diff --git a/packages/overture-schema-pyspark/tests/__init__.py b/packages/overture-schema-pyspark/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/_support/__init__.py b/packages/overture-schema-pyspark/tests/_support/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/_support/harness.py b/packages/overture-schema-pyspark/tests/_support/harness.py new file mode 100644 index 000000000..99681807d --- /dev/null +++ b/packages/overture-schema-pyspark/tests/_support/harness.py @@ -0,0 +1,306 @@ +"""Validation harness for generated conformance tests. + +Builds a single DataFrame per model type from scenario mutations, +runs validation once, and indexes violations by scenario ID. +""" + +from __future__ import annotations + +import copy +import uuid +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any + +from overture.schema.pyspark.check import Check +from overture.schema.pyspark.validate import evaluate_checks, explain_errors +from pyspark.sql import SparkSession +from pyspark.sql.types import ( + ArrayType, + DataType, + DoubleType, + FloatType, + MapType, + StringType, + StructField, + StructType, +) +from shapely import wkb, wkt + +from .helpers import PathTraversalError, deep_merge +from .scenarios import Scenario + +# Namespace for `_scenario_id` UUIDs. Distinct from +# `overture.schema.codegen.pyspark.test_data.base_row._BASE_ROW_NAMESPACE` +# (which synthesizes model `id` values) so a model `id` can never +# collide with a scenario tag and confuse the violations index. +_NAMESPACE = uuid.UUID("a1b2c3d4-e5f6-7890-abcd-ef1234567890") + + +@dataclass(frozen=True) +class ValidationResults: + """Named return type from run_validation_pipeline.""" + + violations: dict[str, set[tuple[str, str]]] + skipped: dict[str, str] + + +def scenario_uuid(scenario_id: str) -> str: + """Deterministic UUID for the harness's `_scenario_id` tag.""" + return str(uuid.uuid5(_NAMESPACE, scenario_id)) + + +def build_scenario_map( + scenarios: Sequence[Scenario], + *, + model_name: str, +) -> dict[str, str]: + """Map _scenario_id values to human-readable scenario IDs. + + Parameters + ---------- + scenarios + All scenarios for a model type. + model_name + Model name for the baseline row ID. + + Returns + ------- + dict[str, str] + Maps _scenario_id UUID string -> scenario ID. Includes baseline. + + Raises + ------ + ValueError + If two scenarios would produce the same UUID key. + """ + baseline_id = f"{model_name}::baseline" + scenario_map: dict[str, str] = {scenario_uuid(baseline_id): baseline_id} + + for s in scenarios: + for suffix in ("::valid", "::invalid"): + label = f"{s.id}{suffix}" + key = scenario_uuid(label) + if key in scenario_map: + raise ValueError( + f"Duplicate scenario id {key!r}: {scenario_map[key]!r} and {label!r}" + ) + scenario_map[key] = label + + return scenario_map + + +def build_scenario_rows( + base_row: dict[str, Any], + scenarios: Sequence[Scenario], + *, + model_name: str, +) -> tuple[list[dict[str, Any]], dict[str, str], dict[str, str]]: + """Build mutation rows and scenario mapping from scenarios. + + Parameters + ---------- + base_row + Valid base row dict from the example loader. + scenarios + Scenarios to apply. + model_name + Model name for baseline ID and UUID namespace. + + Returns + ------- + tuple + (rows, scenario_map, skipped) where rows is a list of row dicts, + scenario_map maps _scenario_id values to scenario IDs, and skipped + maps scenario IDs to skip reasons. + """ + scenario_map = build_scenario_map(scenarios, model_name=model_name) + base_row = sanitize_row(base_row) + # Deep-copy every row so nested structures aren't aliased with base_row; + # a future in-place mutation of one row would otherwise leak across rows. + rows: list[dict[str, Any]] = [ + { + **copy.deepcopy(base_row), + "_scenario_id": scenario_uuid(f"{model_name}::baseline"), + } + ] + skipped: dict[str, str] = {} + + for s in scenarios: + try: + invalid_row = sanitize_row(s.mutate(deep_merge(base_row, s.scaffold))) + invalid_row["_scenario_id"] = scenario_uuid(f"{s.id}::invalid") + # The valid row exercises a real value at the check's target: it + # merges the scaffold (a constraint-satisfying structure reaching + # the target) onto the base row, with NO mutation. A scenario may + # override with `valid_scaffold` to seed a specific value -- e.g. + # the literal alternative of an `X | Literal[c]` field. Without a + # valid scaffold the assertion would be vacuous: a target reachable + # only through scaffolded nesting is absent from a plain base-row + # copy, so a check that wrongly rejects a valid value passes green. + valid_source = ( + s.valid_scaffold if s.valid_scaffold is not None else s.scaffold + ) + valid_row = sanitize_row(deep_merge(base_row, valid_source)) + valid_row["_scenario_id"] = scenario_uuid(f"{s.id}::valid") + rows.append(valid_row) + rows.append(invalid_row) + except PathTraversalError as e: + skipped[s.id] = str(e) + + return rows, scenario_map, skipped + + +_WKT_PREFIXES = ( + "POINT", + "LINESTRING", + "POLYGON", + "MULTIPOINT", + "MULTILINESTRING", + "MULTIPOLYGON", + "GEOMETRYCOLLECTION", +) + +# Schema field whose string value should be parsed as WKT and re-emitted as +# WKB (the storage representation Spark's BinaryType expects). +_GEOMETRY_FIELD = "geometry" + + +def sanitize_row(row: dict[str, Any]) -> dict[str, Any]: + """Return a deep copy of `row` with WKT geometry strings converted to WKB. + + Geometry values from TOML examples are WKT strings, but the schema + expects BinaryType (WKB). Walks the row recursively; any string at + the `geometry` key that looks like WKT is converted via shapely. + """ + return _sanitize_in_place(copy.deepcopy(row)) + + +def _sanitize_in_place(d: dict[str, Any]) -> dict[str, Any]: + for key, value in d.items(): + if isinstance(value, dict): + d[key] = _sanitize_in_place(value) + elif isinstance(value, list): + d[key] = [ + _sanitize_in_place(item) if isinstance(item, dict) else item + for item in value + ] + elif ( + key == _GEOMETRY_FIELD + and isinstance(value, str) + and value.upper().startswith(_WKT_PREFIXES) + ): + d[key] = wkb.dumps(wkt.loads(value)) + return d + + +def assert_schema_covers_checks(schema: StructType, checks: list[Check]) -> None: + """Assert every column a check reads exists in the schema. + + Covers field and model-level checks alike: each top-level column in a + check's `read_columns` must be a schema column. This is a fast sanity + check; deeper field paths are the codegen's responsibility and surface + at Spark execution time. + """ + top_level = {f.name for f in schema.fields} + for chk in checks: + missing = chk.read_columns - top_level + if missing: + raise AssertionError( + f"Check reads columns {sorted(missing)} " + f"not found in schema. Available: {sorted(top_level)}" + ) + + +_FLOAT_TYPES = (DoubleType, FloatType) + + +def coerce_to_schema(value: Any, dtype: DataType) -> Any: + """Cast Python ints to floats where the schema declares a float column. + + A discriminated union widens a numeric field to the broadest member type + (e.g. a `uint8` value alongside `float64` values becomes a `DoubleType` + column). A scaffold built for the narrow arm carries a Python `int`, which + Spark stores as null in a `DoubleType` column (`createDataFrame` does not + coerce with `verifySchema=False`) -- a null that fires `required` on the + `::valid` row. Recursing the row against the schema aligns each numeric + value with its declared column type, so a valid row stays valid. `bool` is + excluded (it is an `int` subclass but maps to `BooleanType`). + + The struct branch keeps only keys the schema declares, mirroring how + `createDataFrame` reads a dict by field name -- so this also drops any + key absent from `dtype`. No row carries such keys today; the filtering is + incidental, not a guarantee. + """ + if value is None: + return None + if isinstance(dtype, StructType) and isinstance(value, dict): + return { + f.name: coerce_to_schema(value[f.name], f.dataType) + for f in dtype.fields + if f.name in value + } + if isinstance(dtype, ArrayType) and isinstance(value, list): + return [coerce_to_schema(item, dtype.elementType) for item in value] + if isinstance(dtype, MapType) and isinstance(value, dict): + return {k: coerce_to_schema(v, dtype.valueType) for k, v in value.items()} + if ( + isinstance(dtype, _FLOAT_TYPES) + and isinstance(value, int) + and not isinstance(value, bool) + ): + return float(value) + return value + + +def run_validation_pipeline( + spark: SparkSession, + schema: StructType, + checks: list[Check], + base_row: dict[str, Any], + scenarios: Sequence[Scenario], + model_name: str, +) -> ValidationResults: + """Run the full validation pipeline. + + Returns a ValidationResults with violations indexed by scenario ID and + a skipped dict for scenarios that could not be built due to path + traversal errors. + """ + assert_schema_covers_checks(schema, checks) + rows, scenario_map, skipped = build_scenario_rows( + base_row, scenarios, model_name=model_name + ) + augmented_schema = StructType( + schema.fields + [StructField("_scenario_id", StringType(), True)] + ) + rows = [coerce_to_schema(row, augmented_schema) for row in rows] + df = spark.createDataFrame(rows, schema=augmented_schema, verifySchema=False) # type: ignore[union-attr] + violations = explain_errors(evaluate_checks(df, checks), checks) + indexed = violations.select("_scenario_id", "field", "check") + return ValidationResults( + violations=index_violations(indexed.collect(), scenario_map), + skipped=skipped, + ) + + +def index_violations( + violation_rows: list[Any], + scenario_map: dict[str, str], +) -> dict[str, set[tuple[str, str]]]: + """Index collected violation rows by human-readable scenario ID. + + Parameters + ---------- + violation_rows + Collected rows from `explain().collect()`. + scenario_map + Mapping from _scenario_id values to scenario IDs. + """ + result: dict[str, set[tuple[str, str]]] = {} + for row in violation_rows: + scenario_id = scenario_map.get(row["_scenario_id"]) + if scenario_id is None: + continue + result.setdefault(scenario_id, set()).add((row["field"], row["check"])) + return result diff --git a/packages/overture-schema-pyspark/tests/_support/helpers.py b/packages/overture-schema-pyspark/tests/_support/helpers.py new file mode 100644 index 000000000..3905f8cea --- /dev/null +++ b/packages/overture-schema-pyspark/tests/_support/helpers.py @@ -0,0 +1,135 @@ +"""Low-level utilities for the conformance test harness. + +Internal to the harness — not imported directly by generated test files. +""" + +from __future__ import annotations + +import copy +from collections.abc import Callable +from typing import Any + +from overture.schema.system.field_path import ArraySegment, FieldPath, coerce + + +def deep_merge(base: dict, scaffold: dict) -> dict: + """Recursively merge scaffold onto a deep copy of base. + + Dict values merge recursively. All other values (including lists) + in scaffold replace the corresponding base values; scaffold values + are deep-copied so callers cannot accidentally share state with + the merged result. Keys present in base but absent from scaffold + are preserved. + """ + result = copy.deepcopy(base) + for key, value in scaffold.items(): + if key in result and isinstance(result[key], dict) and isinstance(value, dict): + result[key] = deep_merge(result[key], value) + else: + result[key] = copy.deepcopy(value) + return result + + +class PathTraversalError(Exception): + """Raised when path traversal cannot proceed.""" + + +def _scaffold_struct(target: dict, name: str) -> dict: + """Return `target[name]` as a dict, scaffolding `{}` when missing or None.""" + child = target.get(name) if isinstance(target, dict) else None + if child is None: + child = {} + target[name] = child + return child + + +def _scaffold_array(target: dict, name: str, path: FieldPath | str) -> list: + """Return `target[name]` as a list, scaffolding `[{}]` when None. + + Empty arrays raise — there is no element to mutate. + """ + child = target.get(name) if isinstance(target, dict) else None + if child is None: + child = [{}] + target[name] = child + if not isinstance(child, list): + raise PathTraversalError( + f"Expected list at '{name}' in path '{path}', got {type(child).__name__}" + ) + if len(child) == 0: + raise PathTraversalError(f"Empty array at '{name}' in path '{path}'") + return child + + +def _descend_through_array( + segment: ArraySegment, target: dict, path: FieldPath | str +) -> list: + """Enter an array segment and walk through its `iter_count`. + + Scaffolds `[{}]` at the outer level when None; deeper levels + (`iter_count > 1`) must already be lists — scaffolding into + nested-list shapes isn't supported because no current schema + needs it. + + Returns the innermost list. For terminal use, write to `[0]`; + for intermediate use, the next segment lives in `[0]`. + """ + container = _scaffold_array(target, segment.name, path) + for _ in range(segment.iter_count - 1): + if len(container) == 0 or not isinstance(container[0], list): + raise PathTraversalError( + f"Expected non-empty nested list at '{segment.name}' in path '{path}'" + ) + container = container[0] + return container + + +def set_at_path(path: FieldPath | str, value: object) -> Callable[[dict], dict]: + """Return a mutator that sets *value* at *path* in a deep copy of the row. + + `[]` always indexes element 0 — one bad element suffices to trigger + a violation since `validate()` checks are element-wise. + + None at an intermediate struct segment is scaffolded as `{}`; None at + an intermediate array segment is scaffolded as `[{}]`. Empty arrays + raise `PathTraversalError` when called — there is no element to mutate. + + Parameters + ---------- + path + A `FieldPath` or its canonical encoded form (`"rules[].tags[].v"`). + value + The value to set at the resolved path. + + Returns + ------- + Callable[[dict], dict] + A function that takes a row dict and returns a deep copy with the + value at `path` replaced. + + Raises + ------ + PathTraversalError + When the path is empty, or when an intermediate or final array + segment is empty (raised at call time, not at factory time). + """ + segments = coerce(path).segments + + def mutator(row_dict: dict) -> dict: + if not segments: + raise PathTraversalError(f"Empty path: {path!r}") + result = copy.deepcopy(row_dict) + target: Any = result + for segment in segments[:-1]: + if isinstance(segment, ArraySegment): + target = _descend_through_array(segment, target, path)[0] + else: + target = _scaffold_struct(target, segment.name) + last = segments[-1] + if isinstance(last, ArraySegment): + _descend_through_array(last, target, path)[0] = value + else: + target[last.name] = value + return result + + return mutator diff --git a/packages/overture-schema-pyspark/tests/_support/mutations.py b/packages/overture-schema-pyspark/tests/_support/mutations.py new file mode 100644 index 000000000..6695ce104 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/_support/mutations.py @@ -0,0 +1,518 @@ +"""Model-level mutation functions for generated conformance tests. + +Each function takes a row dict and returns a modified copy that should +trigger a specific model-level constraint violation. Generated test +files import these by name. +""" + +from __future__ import annotations + +import copy +from collections.abc import Callable +from typing import Any + +from overture.schema.system.field_path import ( + ArraySegment, + FieldPath, + FieldSegment, + ScalarPath, + coerce, +) + +from .helpers import PathTraversalError + +_SENTINEL = "__FORBIDDEN_PRESENT__" +_NOT_EQUAL_PREFIX = "__NOT_" +_STUB_MAP_KEY = "_stub" + + +def mutate_require_any_of( + row_dict: dict, + field_names: list[FieldPath | str], + *, + array_path: FieldPath | str | None = None, + struct_path: str | None = None, + map_path: FieldPath | str | None = None, +) -> dict: + """Null every named field so `require_any_of` fires. + + Parameters + ---------- + array_path + Array column the constrained model lives inside. When None, the + fields live at the row root. + struct_path + Optional single intermediate struct field between the array + element (or map value) and the target fields. + map_path + Map column whose `dict[K, Model]` value carries the constraint. + Mutually exclusive with `array_path`. + + See `_null_all_named_fields` for the full nesting semantics. + """ + return _null_all_named_fields( + row_dict, + field_names, + array_path=array_path, + struct_path=struct_path, + map_path=map_path, + ) + + +def mutate_radio_group(row_dict: dict, field_names: list[FieldPath | str]) -> dict: + """Set first two fields to True so radio_group fires.""" + result = copy.deepcopy(row_dict) + for name in field_names[:2]: + _set_nested(result, name, True) + return result + + +def mutate_require_any_true(row_dict: dict, disabling_values: dict[str, Any]) -> dict: + """Set each condition field to a value that makes its condition false. + + `require_any_true` fires only when no condition is true, so writing a + value each condition rejects (e.g. `is_land=False` for + `FieldEqCondition("is_land", True)`) makes them all false. The renderer + computes the per-field disabling value from the constraint's conditions. + """ + result = copy.deepcopy(row_dict) + for name, value in disabling_values.items(): + _set_nested(result, name, value) + return result + + +def mutate_min_fields_set( + row_dict: dict, + field_names: list[FieldPath | str], + *, + array_path: FieldPath | str | None = None, + struct_path: str | None = None, + map_path: FieldPath | str | None = None, +) -> dict: + """Null every named field so `min_fields_set(N)` fires (0 < N). + + The descriptor enumerates every field of the constrained model, so + nulling all of them drops the non-null count to zero -- below any + positive `count`. Nulling required fields incidentally trips their + `check_required` checks; the conformance test only asserts the + expected violation is present, so the extra failures don't matter. + + `array_path` / `struct_path` / `map_path` mirror `mutate_require_any_of` + for the case where the constrained model is reached through array or map + iteration (and optionally one intermediate struct field). + """ + return _null_all_named_fields( + row_dict, + field_names, + array_path=array_path, + struct_path=struct_path, + map_path=map_path, + ) + + +def _null_all_named_fields( + row_dict: dict, + field_names: list[FieldPath | str], + *, + array_path: FieldPath | str | None, + struct_path: str | None, + map_path: FieldPath | str | None = None, +) -> dict: + """Return a deep copy of *row_dict* with every named field set to None. + + Without *array_path* or *map_path*, the fields live at the row root. + With *array_path*, the fields live inside elements of that array column; + with *map_path*, inside the value model of that map column. *struct_path* + names an optional single intermediate struct field between the array + element / map value and the target fields. A null array is replaced with + a single stub element so the violation has a row to fire on; a null map + is stubbed analogously. + """ + result = copy.deepcopy(row_dict) + if map_path is not None: + target = _map_value_to_mutate(result, map_path) + if struct_path: + target = _scaffold_struct_child(target, struct_path) + for name in field_names: + _set_nested(target, name, None) + return result + if array_path is None: + for name in field_names: + _set_nested(result, name, None) + return result + + arr: list[dict] | None = _get_nested(result, array_path) # type: ignore[assignment] + if arr is None: + stub: dict = {} + for name in field_names: + _set_nested(stub, name, None, create=True) + element = {struct_path: stub} if struct_path else stub + _set_nested(result, array_path, [element]) + else: + for element in arr: + target = ( + _scaffold_struct_child(element, struct_path) if struct_path else element + ) + for name in field_names: + _set_nested(target, name, None) + return result + + +def _scaffold_struct_child(parent: dict, name: str) -> dict: + """Return `parent[name]` as a dict, scaffolding `{}` when missing or None.""" + child = parent.get(name) + if child is None: + child = {} + parent[name] = child + return child + + +def _map_value_to_mutate(row: dict, map_path: FieldPath | str) -> dict: + """Return the value model of the map at *map_path*, stubbing if absent. + + A model-level constraint on a `dict[K, Model]` value targets the value + model. The base row supplies the map's single entry; a missing or empty + map is stubbed with one entry so the violation has a value to fire on -- + mirroring how a null array is stubbed in `_null_all_named_fields`. The + map carries exactly one entry (base-row generation emits a single + key/value pair), so the sole value is unambiguous. + """ + m = _get_nested(row, map_path) + if isinstance(m, dict) and m: + return next(iter(m.values())) + stub: dict = {} + _set_nested(row, map_path, {_STUB_MAP_KEY: stub}, create=True) + return stub + + +def mutate_require_if( + row_dict: dict, + field_names: list[FieldPath | str], + condition_field: FieldPath | str, + condition_value: object, + *, + negate: bool = False, + array_path: FieldPath | str | None = None, + inner_array_path: FieldPath | str | None = None, + map_path: FieldPath | str | None = None, +) -> dict: + """Set condition to trigger require_if, then null target fields.""" + result = copy.deepcopy(row_dict) + + def _apply(target: dict) -> None: + _ensure_condition(target, condition_field, condition_value, negate=negate) + for name in field_names: + _set_nested(target, name, None) + + _apply_to_targets(result, _apply, array_path, inner_array_path, map_path) + return result + + +def mutate_forbid_if( + row_dict: dict, + field_names: list[str], + condition_field: FieldPath | str, + condition_value: object, + *, + negate: bool = False, + fill_values: dict[str, object] | None = None, + array_path: FieldPath | str | None = None, + inner_array_path: FieldPath | str | None = None, + map_path: FieldPath | str | None = None, +) -> dict: + """Set condition to trigger forbid_if, ensure target fields are non-null. + + `field_names` are flat scalar field names — model-level forbid_if + references fields by name on the enclosing model. `fill_values` is + keyed by the same names. + """ + result = copy.deepcopy(row_dict) + fills = fill_values or {} + + def _apply(target: dict) -> None: + _ensure_condition(target, condition_field, condition_value, negate=negate) + for name in field_names: + if _get_nested(target, name) is None: + _set_nested(target, name, fills.get(name, _SENTINEL)) + + _apply_to_targets(result, _apply, array_path, inner_array_path, map_path) + return result + + +def mutate_unique_items(row_dict: dict, path: FieldPath | str) -> dict: + """Duplicate the first array element so unique_items fires. + + Supports bracket paths like `"restrictions[].when.mode"` -- enters + element 0 at each `[]` segment, then duplicates the first element + of the final array. A terminal `[]` (e.g. `"hierarchies[]"`) + targets the inner array at element 0 of the named field -- the + walker descends one extra level per bracket on the terminal + segment and duplicates the first element of the array it lands on. + """ + result = copy.deepcopy(row_dict) + segments = coerce(path).segments + + parent: Any = _walk_strict(result, path, segments[:-1]) + last = segments[-1] + if not isinstance(parent, dict) or last.name not in parent: + raise PathTraversalError(f"Missing key '{last.name}' in path '{path}'") + + # When the terminal is an array segment, descend `iter_count` levels of + # `[0]`. Otherwise the terminal struct already references the list to + # mutate. The final `container[key]` must itself be a list. + container: Any + key: int | str + if isinstance(last, ArraySegment): + container, key = _descend_iter_count( + parent[last.name], last.iter_count, last.name, path + ) + else: + container = parent + key = last.name + arr = container[key] + if not isinstance(arr, list): + raise PathTraversalError( + f"Expected list at terminal of path '{path}', got {type(arr).__name__}" + ) + _duplicate_first(container, key, arr) + return result + + +def mutate_map_key(row_dict: dict, path: FieldPath | str, bad_key: object) -> dict: + """Replace the single map entry's key with *bad_key*, preserving its value. + + *path* is the struct-field path to the map column (e.g. `"names.common"`). + The base row / scaffold populates the map with one valid entry, so a + one-entry replacement suffices to trigger the key check. Raises + `PathTraversalError` when the map is missing or empty. + """ + _key, value = _single_map_entry(row_dict, path) + return _replace_map(row_dict, path, {bad_key: value}) + + +def mutate_map_value(row_dict: dict, path: FieldPath | str, bad_value: object) -> dict: + """Replace the single map entry's value with *bad_value*, preserving its key. + + Mirror of `mutate_map_key` for the value side. Raises + `PathTraversalError` when the map is missing or empty. + """ + key, _value = _single_map_entry(row_dict, path) + return _replace_map(row_dict, path, {key: bad_value}) + + +def _single_map_entry(row_dict: dict, path: FieldPath | str) -> tuple[Any, Any]: + """Return the `(key, value)` of the sole entry of the map at *path*.""" + m = _get_nested(row_dict, path) + if not isinstance(m, dict) or not m: + raise PathTraversalError(f"Missing or empty map at path '{path}'") + return next(iter(m.items())) + + +def _replace_map(row_dict: dict, path: FieldPath | str, new_map: dict) -> dict: + """Return a deep copy of *row_dict* with the map at *path* replaced.""" + result = copy.deepcopy(row_dict) + _set_nested(result, path, new_map) + return result + + +def _walk_strict( + target: Any, path: FieldPath | str, segments: tuple[FieldSegment, ...] | None = None +) -> Any: + """Walk *path* without scaffolding, raising on missing or null nodes. + + Raises `PathTraversalError` on missing or null struct intermediates, + and on empty arrays encountered at array segments (each `[]` in a + segment's `iter_count` descends one element, which requires a + non-empty list). When *segments* is provided it overrides the + segments derived from *path*; *path* still labels error messages. + """ + if segments is None: + segments = coerce(path).segments + for segment in segments: + if not isinstance(target, dict) or target.get(segment.name) is None: + raise PathTraversalError( + f"Missing or null key '{segment.name}' in path '{path}'" + ) + target = target[segment.name] + if isinstance(segment, ArraySegment): + container, key = _descend_iter_count( + target, segment.iter_count, segment.name, path + ) + target = container[key] + return target + + +def _descend_iter_count( + arr: list, iter_count: int, name: str, path: FieldPath | str +) -> tuple[Any, int]: + """Descend *iter_count* levels into nested lists via element 0. + + Each level requires a non-empty list; the error label for depth `d` + is *name* followed by `d` `[]` markers. Returns the final + `(container, key)` write site so callers can read (`container[key]`) + or replace (`container[key] = ...`) the innermost element. + """ + container: Any = [arr] + key = 0 + for depth in range(iter_count): + inner = container[key] + _require_non_empty_array(inner, f"{name}{'[]' * depth}", path) + container, key = inner, 0 + return container, key + + +def _require_non_empty_array(value: Any, name: str, path: FieldPath | str) -> None: + """Raise `PathTraversalError` unless *value* is a non-empty list.""" + if not isinstance(value, list) or len(value) == 0: + raise PathTraversalError(f"Empty or missing array at '{name}' in path '{path}'") + + +def _duplicate_first(container: Any, key: int | str, arr: list) -> None: + """Replace `container[key]` with `arr` having its first element duplicated. + + No-op when `arr` is empty. Both elements are deep-copied so callers + cannot accidentally share state between the duplicates. + """ + if not arr: + return + dup = copy.deepcopy(arr[0]) + container[key] = [dup, copy.deepcopy(dup)] + list(arr[1:]) + + +_Applicator = Callable[[dict], None] + + +def _apply_to_targets( + row: dict, + fn: _Applicator, + array_path: FieldPath | str | None, + inner_array_path: FieldPath | str | None, + map_path: FieldPath | str | None = None, +) -> None: + """Apply a mutation function to target dicts at the appropriate nesting level. + + Without array or map paths, applies directly to the row. With + `array_path`, iterates over elements of that array. With both + `array_path` and `inner_array_path`, iterates over outer elements, + navigates the inner struct path to a nested array, then iterates those + elements. With `map_path`, applies to the value model of that map column + (stubbing one entry when the map is absent). + + Creates stub array elements when the arrays are null so the mutation + can populate them. + """ + if map_path is not None: + fn(_map_value_to_mutate(row, map_path)) + return + if array_path is None: + fn(row) + return + outer_arr: list[dict] | None = _get_nested(row, array_path) # type: ignore[assignment] + if outer_arr is None: + outer_stub: dict = {} + _stub_apply(outer_stub, inner_array_path, fn) + _set_nested(row, array_path, [outer_stub]) + return + if inner_array_path is None: + for element in outer_arr: + fn(element) + else: + for element in outer_arr: + inner_arr: list[dict] | None = _get_nested(element, inner_array_path) # type: ignore[assignment] + if inner_arr is not None: + for inner_element in inner_arr: + fn(inner_element) + else: + _stub_apply(element, inner_array_path, fn) + + +def _stub_apply( + parent: dict, + inner_array_path: FieldPath | str | None, + fn: _Applicator, +) -> None: + """Build a stub element at `inner_array_path` inside *parent* and run `fn`. + + When `inner_array_path` is None, *parent* itself is the stub that + `fn` mutates. Otherwise an empty stub is inserted as the sole + element of `[stub]` at `inner_array_path` inside *parent* + (scaffolding intermediate dicts), and `fn` mutates the stub. + """ + if inner_array_path is None: + fn(parent) + return + stub: dict = {} + fn(stub) + _set_nested(parent, inner_array_path, [stub], create=True) + + +def _ensure_condition( + d: dict, + condition_field: FieldPath | str, + condition_value: object, + *, + negate: bool, +) -> None: + """Set condition_field so the constraint condition evaluates to True. + + When *negate* is False, sets the field to *condition_value* (the + condition is `field == value`). When True, ensures the field is + NOT equal to *condition_value* (the condition is `field != value`); + if it already differs, leaves it alone. + """ + if negate: + current = _get_nested(d, condition_field) + if current == condition_value: + _set_nested(d, condition_field, f"{_NOT_EQUAL_PREFIX}{condition_value}__") + else: + _set_nested(d, condition_field, condition_value) + + +def _as_scalar_path(path: FieldPath | str) -> ScalarPath: + """Coerce *path* to a `ScalarPath`, rejecting array or map markers. + + The dict-walking helpers operate only on struct fields; an array or + map-projection marker indicates the caller wanted array-/map-aware + navigation and picked the wrong helper. + """ + coerced = coerce(path) + if not isinstance(coerced, ScalarPath): + raise ValueError(f"struct-only path expected, got {coerced!r} for {path!r}") + return coerced + + +def _set_nested( + d: dict, path: FieldPath | str, value: object, *, create: bool = False +) -> None: + """Set a value in a nested dict using a struct-field path. + + When *create* is True, intermediate dicts are created if missing or + None. When an intermediate is None and *value* is also None, the path + is already effectively null — returns without error. + """ + segments = _as_scalar_path(path).segments + target = d + for segment in segments[:-1]: + part = segment.name + if create and (part not in target or target[part] is None): + target[part] = {} + child = target.get(part) if isinstance(target, dict) else None + if child is None: + if value is None: + return + raise PathTraversalError(f"None intermediate at '{part}' in path '{path}'") + target = child + target[segments[-1].name] = value + + +def _get_nested(d: dict, path: FieldPath | str) -> object: + """Get a value from a nested dict using a struct-field path. + + Returns None when any intermediate key is missing or not a dict. + """ + target: object = d + for segment in _as_scalar_path(path).segments: + if not isinstance(target, dict) or segment.name not in target: + return None + target = target[segment.name] + return target diff --git a/packages/overture-schema-pyspark/tests/_support/registry.py b/packages/overture-schema-pyspark/tests/_support/registry.py new file mode 100644 index 000000000..5108bd4b9 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/_support/registry.py @@ -0,0 +1,45 @@ +"""Registry registration helper for tests. + +Provides a context manager that registers a model type in the runtime +`REGISTRY` and guarantees teardown on exit, even if the test body raises. +""" + +from collections.abc import Callable, Iterator +from contextlib import contextmanager + +from overture.schema.pyspark._registry import REGISTRY +from overture.schema.pyspark.check import Check, ModelValidation +from pyspark.sql.types import StructType + + +@contextmanager +def register_model( + model_type: str, + schema: StructType, + checks: Callable[[], list[Check]], +) -> Iterator[None]: + """Register a model type in `REGISTRY` for the duration of a test. + + Removes `REGISTRY[model_type]` on exit so a failed test body never + leaks an entry into sibling tests. Uses `pop(..., None)` so a body that + already removed or rebound the key does not raise a `KeyError` that + would mask the body's own exception. + + Parameters + ---------- + model_type + The registry key string (e.g. `"_test_cli"`). + schema + StructType to associate with the model type. + checks + Callable returning the list of `Check` objects for the model type. + + Yields + ------ + None + """ + REGISTRY[model_type] = ModelValidation(schema=schema, checks=checks) + try: + yield + finally: + REGISTRY.pop(model_type, None) diff --git a/packages/overture-schema-pyspark/tests/_support/scenarios.py b/packages/overture-schema-pyspark/tests/_support/scenarios.py new file mode 100644 index 000000000..8ccfa0af4 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/_support/scenarios.py @@ -0,0 +1,44 @@ +"""Scenario dataclass for generated conformance tests.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True, slots=True) +class Scenario: + """A test scenario: a mutation that should produce a specific violation. + + Parameters + ---------- + id + Human-readable scenario identifier, e.g. `"building::id:required"`. + scaffold + Dict merged onto the base row before mutation to provide valid values + for fields the base row lacks (e.g. array elements for nested paths). + mutate + Callable applied to `deep_merge(base_row, scaffold)` to produce the + invalid row. Must return a new dict; must not mutate its argument. + expected_field + Field name expected in the violation output. + expected_check + Check name expected in the violation output. + """ + + id: str + scaffold: dict[str, Any] + mutate: Callable[[dict], dict] + expected_field: str + expected_check: str + valid_scaffold: dict[str, Any] | None = None + """Override scaffold for the `::valid` row, when it must differ from `scaffold`. + + The harness builds the `::valid` row from `scaffold` by default (merged + onto the base row, no mutation), which already places a constraint-valid + value at the check's target. Set this only when the valid row needs a + *different* value there -- e.g. an `X | Literal[c]` field, where it seeds + the literal alternative `c` to prove the check accepts it, distinct from + the synthesized `X` value the mutation scaffold carries. + """ diff --git a/packages/overture-schema-pyspark/tests/conftest.py b/packages/overture-schema-pyspark/tests/conftest.py new file mode 100644 index 000000000..ccacb1aac --- /dev/null +++ b/packages/overture-schema-pyspark/tests/conftest.py @@ -0,0 +1,50 @@ +"""Shared pytest fixtures for overture-schema-pyspark tests.""" + +import os +import socket +import sys +from collections.abc import Callable +from typing import Any + +import pytest +from pyspark.sql import SparkSession + +# Ensure PySpark workers use the same Python as the driver to avoid +# version mismatch errors when a different system Python is on PATH. +os.environ.setdefault("PYSPARK_PYTHON", sys.executable) +os.environ.setdefault("PYSPARK_DRIVER_PYTHON", sys.executable) + + +def pytest_configure(config: pytest.Config) -> None: + """Suppress ResourceWarning from PySpark's unclosed py4j sockets. + + PySpark uses py4j to communicate with the JVM. py4j socket proxies + are GC'd between tests and their __del__ fires ResourceWarning via + sys.unraisablehook. With -W error this becomes a test failure. + + The original hook is preserved for all other unraisable exceptions. + """ + original_hook: Callable[[Any], None] = sys.unraisablehook + + def _hook(unraisable: Any) -> None: + if isinstance(unraisable.exc_value, ResourceWarning) and isinstance( + unraisable.object, socket.socket + ): + return + original_hook(unraisable) + + sys.unraisablehook = _hook + + +@pytest.fixture(scope="session") +def spark() -> SparkSession: + """Provide a local SparkSession for testing.""" + session = ( + SparkSession.builder.master("local[1]") + .appName("overture-pyspark-tests") + .config("spark.ui.enabled", "false") + .config("spark.sql.shuffle.partitions", "1") + .getOrCreate() + ) + session.sparkContext.setLogLevel("ERROR") + return session diff --git a/packages/overture-schema-pyspark/tests/expressions/__init__.py b/packages/overture-schema-pyspark/tests/expressions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/expressions/test_column_patterns.py b/packages/overture-schema-pyspark/tests/expressions/test_column_patterns.py new file mode 100644 index 000000000..027598b64 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/expressions/test_column_patterns.py @@ -0,0 +1,400 @@ +"""Tests for column_patterns — structural PySpark composition helpers.""" + +from overture.schema.pyspark.expressions.column_patterns import ( + array_check, + check_struct_unique, + coalesce_errors, + error_msg, + map_keys_check, + map_values_check, + nested_array_check, +) +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_require_any_of, + check_string_min_length, +) +from pyspark.sql import Row, SparkSession +from pyspark.sql import functions as F + + +def test_error_msg_concatenates(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="bad")]) + result = df.select(error_msg("field: got ", F.col("val")).alias("msg")).collect() + assert result[0]["msg"] == "field: got bad" + + +def test_error_msg_multiple_values(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(a="x", b="y")]) + result = df.select( + error_msg("prefix ", F.col("a"), F.lit(" and "), F.col("b")).alias("msg") + ).collect() + assert result[0]["msg"] == "prefix x and y" + + +def test_error_msg_null_value_does_not_nullify_message(spark: SparkSession) -> None: + # A NULL interpolated value must not make the whole message NULL: F.concat + # would, and a NULL message is dropped by array_compact, silently swallowing + # the violation (e.g. an out-of-bounds linear-reference range [null, 1.5]). + # The null must render as a literal instead. + df = spark.createDataFrame([Row(val=None)], schema="val double") + result = df.select(error_msg("got ", F.col("val")).alias("msg")).collect() + assert result[0]["msg"] == "got null" + + +def test_array_check_null_column_returns_null(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(items=None)], + schema="items array>", + ) + result = df.select( + array_check("items", lambda el: F.lit("err")).alias("errs") + ).collect() + assert result[0]["errs"] is None + + +def test_array_check_filters_nulls(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(items=[Row(val="ok"), Row(val="bad")])], + schema="items array>", + ) + result = df.select( + array_check( + "items", + lambda el: F.when(el["val"] == "bad", F.lit("error")), + ).alias("errs") + ).collect() + assert result[0]["errs"] == ["error"] + + +def test_array_check_empty_when_all_valid(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(items=[Row(val="ok")])], + schema="items array>", + ) + result = df.select( + array_check( + "items", + lambda el: F.when(el["val"] == "bad", F.lit("error")), + ).alias("errs") + ).collect() + assert result[0]["errs"] == [] + + +def test_struct_unique_no_duplicates(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(items=[Row(id="a"), Row(id="b")])], + schema="items array>", + ) + result = df.select(check_struct_unique("items").alias("err")).collect() + assert result[0]["err"] is None + + +def test_struct_unique_with_duplicates(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(items=[Row(id="a"), Row(id="a")])], + schema="items array>", + ) + result = df.select(check_struct_unique("items").alias("err")).collect() + assert result[0]["err"] is not None + assert "duplicate" in result[0]["err"] + + +def test_struct_unique_null_column(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(items=None)], + schema="items array>", + ) + result = df.select(check_struct_unique("items").alias("err")).collect() + assert result[0]["err"] is None + + +def test_struct_unique_repeated_value_different_fields(spark: SparkSession) -> None: + """Structs with same value subfield but different other fields are unique.""" + df = spark.createDataFrame( + [ + Row( + items=[ + Row(value="a", pos=0.0), + Row(value="b", pos=0.5), + Row(value="a", pos=0.7), + ] + ) + ] + ) + result = df.select(check_struct_unique("items").alias("err")).collect() + assert result[0]["err"] is None + + +def test_struct_unique_single_element(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(items=[Row(id="a")])], + schema="items array>", + ) + result = df.select(check_struct_unique("items").alias("err")).collect() + assert result[0]["err"] is None + + +def test_array_check_accepts_column(spark: SparkSession) -> None: + """array_check works when passed a Column instead of a string name.""" + df = spark.createDataFrame( + [Row(items=[Row(val="ok"), Row(val="bad")])], + schema="items array>", + ) + result = df.select( + array_check( + F.col("items"), + lambda el: F.when(el["val"] == "bad", F.lit("error")), + ).alias("errs") + ).collect() + assert result[0]["errs"] == ["error"] + + +def test_check_struct_unique_accepts_column(spark: SparkSession) -> None: + """check_struct_unique works when passed a Column instead of a string name.""" + df = spark.createDataFrame( + [Row(items=[Row(id="a"), Row(id="a")])], + schema="items array>", + ) + result = df.select(check_struct_unique(F.col("items")).alias("err")).collect() + assert result[0]["err"] is not None + assert "duplicate" in result[0]["err"] + + +def test_check_struct_unique_column_null(spark: SparkSession) -> None: + """check_struct_unique with Column input handles null.""" + df = spark.createDataFrame( + [Row(items=None)], schema="items array>" + ) + result = df.select(check_struct_unique(F.col("items")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_nested_array_check_flattens(spark: SparkSession) -> None: + """Inner array_check per outer element produces flat error list.""" + schema = "items array>>" + df = spark.createDataFrame( + [ + Row( + items=[ + Row(tags=["good", "bad"]), + Row(tags=["worse"]), + ] + ) + ], + schema=schema, + ) + result_col = nested_array_check( + "items", + lambda el: array_check( + el["tags"], + lambda tag: F.when(tag != "good", F.concat(F.lit("bad: "), tag)), + ), + ) + result = df.select(coalesce_errors(result_col).alias("errs")).collect() + errors = result[0]["errs"] + assert len(errors) == 2 + assert all(isinstance(e, str) for e in errors) + + +def test_nested_array_check_null_outer(spark: SparkSession) -> None: + schema = "items array>>" + df = spark.createDataFrame([Row(items=None)], schema=schema) + result_col = nested_array_check( + "items", + lambda el: array_check( + el["tags"], + lambda tag: F.when(tag != "good", F.lit("bad")), + ), + ) + result = df.select(coalesce_errors(result_col).alias("errs")).collect() + assert result[0]["errs"] == [] + + +def test_nested_array_check_mixed_null_inner_with_sibling_errors( + spark: SparkSession, +) -> None: + """A null inner array must not nullify sibling errors during flatten. + + `F.flatten` returns NULL whenever any sub-array is NULL. Without + guarding inner nulls, the outer transform produces NULL and every + sibling error is silently dropped. + """ + schema = "items array>>" + df = spark.createDataFrame( + [ + Row( + items=[ + Row(tags=["good"]), + Row(tags=None), + Row(tags=["bad"]), + ] + ) + ], + schema=schema, + ) + result_col = nested_array_check( + "items", + lambda el: array_check( + el["tags"], + lambda tag: F.when(tag != "good", F.concat(F.lit("bad: "), tag)), + ), + ) + result = df.select(coalesce_errors(result_col).alias("errs")).collect() + assert result[0]["errs"] == ["bad: bad"] + + +def test_nested_array_check_no_errors(spark: SparkSession) -> None: + schema = "items array>>" + df = spark.createDataFrame( + [Row(items=[Row(tags=["good"])])], + schema=schema, + ) + result_col = nested_array_check( + "items", + lambda el: array_check( + el["tags"], + lambda tag: F.when(tag != "good", F.lit("bad")), + ), + ) + result = df.select(coalesce_errors(result_col).alias("errs")).collect() + assert result[0]["errs"] == [] + + +def test_map_keys_check_flags_bad_key(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(tags={"good": "v", "bad": "v"})], + schema="tags map", + ) + result = df.select( + map_keys_check("tags", lambda k: F.when(k == "bad", F.lit("bad key"))).alias( + "errs" + ) + ).collect() + assert result[0]["errs"] == ["bad key"] + + +def test_map_values_check_flags_bad_value(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(tags={"a": "ok", "b": "bad"})], + schema="tags map", + ) + result = df.select( + map_values_check( + "tags", lambda v: F.when(v == "bad", F.lit("bad value")) + ).alias("errs") + ).collect() + assert result[0]["errs"] == ["bad value"] + + +def test_map_values_check_descends_into_value_struct_field( + spark: SparkSession, +) -> None: + # A field check on a `dict[str, Model]` value navigates into the value + # struct: map_values_check over a struct-navigating lambda, the exact + # composition the renderer emits for a map-of-model value field. + df = spark.createDataFrame( + [Row(items={"a": Row(label="")})], + schema="items map>", + ) + result = df.select( + map_values_check( + "items", lambda v: check_string_min_length(v["label"], 1) + ).alias("errs") + ).collect() + assert result[0]["errs"] == ["minimum length 1, got 0"] + + +def test_map_values_check_passes_valid_value_struct_field( + spark: SparkSession, +) -> None: + df = spark.createDataFrame( + [Row(items={"a": Row(label="ok")})], + schema="items map>", + ) + result = df.select( + map_values_check( + "items", lambda v: check_string_min_length(v["label"], 1) + ).alias("errs") + ).collect() + assert result[0]["errs"] == [] + + +def test_map_values_check_enforces_value_model_constraint( + spark: SparkSession, +) -> None: + # A model-level constraint on a `dict[str, Model]` value: map_values_check + # wrapping check_require_any_of over the value struct's fields, the exact + # composition the renderer emits for a map-of-model value-model constraint. + df = spark.createDataFrame( + [Row(subs={"a": Row(foo=None, bar=None)})], + schema="subs map>", + ) + result = df.select( + map_values_check( + "subs", + lambda v: check_require_any_of([v["foo"], v["bar"]], ["foo", "bar"]), + ).alias("errs") + ).collect() + assert result[0]["errs"] == ["requires at least one of foo, bar"] + + +def test_map_values_check_passes_satisfied_value_model_constraint( + spark: SparkSession, +) -> None: + df = spark.createDataFrame( + [Row(subs={"a": Row(foo=1, bar=None)})], + schema="subs map>", + ) + result = df.select( + map_values_check( + "subs", + lambda v: check_require_any_of([v["foo"], v["bar"]], ["foo", "bar"]), + ).alias("errs") + ).collect() + assert result[0]["errs"] == [] + + +def test_map_keys_check_null_column_returns_null(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(tags=None)], schema="tags map") + result = df.select( + map_keys_check("tags", lambda k: F.lit("err")).alias("errs") + ).collect() + assert result[0]["errs"] is None + + +def test_map_values_check_all_valid_empty(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(tags={"a": "ok"})], schema="tags map" + ) + result = df.select( + map_values_check("tags", lambda v: F.when(v == "bad", F.lit("err"))).alias( + "errs" + ) + ).collect() + assert result[0]["errs"] == [] + + +def test_map_keys_check_accepts_column(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(tags={"bad": "v"})], schema="tags map" + ) + result = df.select( + map_keys_check(F.col("tags"), lambda k: F.when(k == "bad", F.lit("err"))).alias( + "errs" + ) + ).collect() + assert result[0]["errs"] == ["err"] + + +def test_coalesce_errors_null_becomes_empty(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(x=1)]) + result = df.select( + coalesce_errors(F.lit(None).cast("array")).alias("errs") + ).collect() + assert result[0]["errs"] == [] + + +def test_coalesce_errors_preserves_array(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(x=1)]) + result = df.select(coalesce_errors(F.array(F.lit("err"))).alias("errs")).collect() + assert result[0]["errs"] == ["err"] diff --git a/packages/overture-schema-pyspark/tests/expressions/test_constraint_expressions.py b/packages/overture-schema-pyspark/tests/expressions/test_constraint_expressions.py new file mode 100644 index 000000000..def5635f6 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/expressions/test_constraint_expressions.py @@ -0,0 +1,1596 @@ +"""Tests for constraint_expressions — constraint type to Column translation.""" + +import struct + +import pytest +from overture.schema.pyspark.expressions.constraint_expressions import ( + check_array_max_length, + check_array_min_length, + check_bbox_completeness, + check_bbox_lat_ordering, + check_bbox_lat_range, + check_bounds, + check_email, + check_enum, + check_forbid_if, + check_geometry_type, + check_json_pointer, + check_linear_range_bounds, + check_linear_range_length, + check_linear_range_order, + check_min_fields_set, + check_pattern, + check_radio_group, + check_require_any_of, + check_require_any_true, + check_require_if, + check_required, + check_string_max_length, + check_string_min_length, + check_stripped, + check_url_format, + check_url_length, + except_literals, +) +from overture.schema.system.primitive import GeometryType +from pyspark.sql import Row, SparkSession +from pyspark.sql import functions as F +from pyspark.sql.types import DoubleType, StringType, StructField, StructType +from shapely.geometry import LineString, MultiPolygon, Point, Polygon + + +def _except_literals_error(spark: SparkSession, value: str | None) -> str | None: + """Run `except_literals` over `check_url_format` for one string value.""" + df = spark.createDataFrame( + [Row(val=value)], schema=StructType([StructField("val", StringType(), True)]) + ) + col = F.col("val") + expr = except_literals(col, check_url_format(col), [""]) + # Spark Row field access is untyped (Any); the column holds an error string. + return df.select(expr.alias("err")).collect()[0]["err"] # type: ignore[no-any-return] + + +def test_except_literals_suppresses_allowed_literal(spark: SparkSession) -> None: + # "" is an allowed literal alternative -> the url_format error is suppressed. + assert _except_literals_error(spark, "") is None + + +def test_except_literals_passes_through_real_violation(spark: SparkSession) -> None: + # A non-literal invalid value still surfaces the inner check's error. + assert _except_literals_error(spark, "not a url") is not None + + +def test_except_literals_passes_through_valid_value(spark: SparkSession) -> None: + assert _except_literals_error(spark, "https://example.com/x") is None + + +def test_except_literals_null_is_not_an_error(spark: SparkSession) -> None: + assert _except_literals_error(spark, None) is None + + +def test_check_bounds_ge_le_valid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=5)]) + result = df.select(check_bounds(F.col("val"), ge=1, le=10).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_bounds_ge_violation(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=0)]) + result = df.select(check_bounds(F.col("val"), ge=1).alias("err")).collect() + assert result[0]["err"] is not None + assert ">= 1" in result[0]["err"] + + +def test_check_bounds_gt_violation(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=0)]) + result = df.select(check_bounds(F.col("val"), gt=0).alias("err")).collect() + assert result[0]["err"] is not None + assert "> 0" in result[0]["err"] + + +def test_check_bounds_le_violation(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=100)]) + result = df.select(check_bounds(F.col("val"), le=50).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_bounds_null_passthrough(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=None)], schema="val int") + result = df.select(check_bounds(F.col("val"), ge=1).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_bounds_nan_lower_bound_violation(spark: SparkSession) -> None: + """NaN satisfies no Pydantic bound, but Spark sorts NaN above all values, + so a lower bound (NaN < v) never fires. check_bounds must reject it.""" + df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") + result = df.select(check_bounds(F.col("val"), ge=0).alias("err")).collect() + assert result[0]["err"] is not None + assert "NaN" in result[0]["err"] + + +def test_check_bounds_nan_gt_violation(spark: SparkSession) -> None: + """Same lower-bound leak as ge, via the strict-greater comparison.""" + df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") + result = df.select(check_bounds(F.col("val"), gt=0).alias("err")).collect() + assert result[0]["err"] is not None + assert "NaN" in result[0]["err"] + + +def test_check_bounds_nan_upper_bound_violation(spark: SparkSession) -> None: + """An upper bound already rejects NaN in Spark (NaN > v is true); the + explicit NaN check keeps that behavior.""" + df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") + result = df.select(check_bounds(F.col("val"), le=1).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_bounds_nan_no_bounds_passes(spark: SparkSession) -> None: + """With no bounds there is nothing to violate; NaN passes, matching + Pydantic's allow_inf_nan default for unconstrained floats.""" + df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") + result = df.select(check_bounds(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_bounds_valid_float_passes(spark: SparkSession) -> None: + """A finite in-range float is unaffected by the NaN guard.""" + df = spark.createDataFrame([Row(val=0.5)], schema="val double") + result = df.select(check_bounds(F.col("val"), ge=0, le=1).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_bounds_nan_guard_off_passes_nan(spark: SparkSession) -> None: + """With check_nan=False the NaN guard is absent; NaN slips past a lower bound.""" + df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") + result = df.select( + check_bounds(F.col("val"), ge=0, check_nan=False).alias("err") + ).collect() + assert result[0]["err"] is None + + +def test_check_bounds_nan_guard_on_rejects_nan(spark: SparkSession) -> None: + """With check_nan=True (default) NaN is rejected even with a lower bound.""" + df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") + result = df.select( + check_bounds(F.col("val"), ge=0, check_nan=True).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "NaN" in result[0]["err"] + + +def test_check_bounds_integer_column_rejects_violation(spark: SparkSession) -> None: + """check_nan=False is safe for integer columns; bound violations still fire.""" + df = spark.createDataFrame([Row(val=0)], schema="val int") + result = df.select( + check_bounds(F.col("val"), ge=1, check_nan=False).alias("err") + ).collect() + assert result[0]["err"] is not None + + +def test_check_bounds_integer_column_accepts_valid(spark: SparkSession) -> None: + """check_nan=False on an integer column: in-bound values pass.""" + df = spark.createDataFrame([Row(val=5)], schema="val int") + result = df.select( + check_bounds(F.col("val"), ge=1, le=10, check_nan=False).alias("err") + ).collect() + assert result[0]["err"] is None + + +def test_check_enum_valid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="road")]) + result = df.select( + check_enum(F.col("val"), ["road", "rail", "water"]).alias("err") + ).collect() + assert result[0]["err"] is None + + +def test_check_enum_invalid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="sky")]) + result = df.select( + check_enum(F.col("val"), ["road", "rail", "water"]).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "sky" in result[0]["err"] + + +class TestCheckPattern: + def test_valid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("AB",)], ["v"]) + result = df.select( + check_pattern(F.col("v"), r"^[A-Z]{2}$", label="test pattern").alias("e") + ) + assert result.collect()[0]["e"] is None + + def test_invalid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("abc",)], ["v"]) + result = df.select( + check_pattern(F.col("v"), r"^[A-Z]{2}$", label="test pattern").alias("e") + ) + err = result.collect()[0]["e"] + assert "invalid test pattern" in err + assert "abc" in err + + def test_null_passes(self, spark: SparkSession) -> None: + df = spark.createDataFrame([(None,)], schema="v string") + result = df.select( + check_pattern(F.col("v"), r"^[A-Z]{2}$", label="test pattern").alias("e") + ) + assert result.collect()[0]["e"] is None + + +class TestCheckMinLength: + def test_at_limit(self, spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(items=["a", "b"])], schema="items array" + ) + result = df.select( + check_array_min_length(F.col("items"), 2).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_below_limit(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(items=["a"])], schema="items array") + result = df.select( + check_array_min_length(F.col("items"), 2).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "minimum length 2" in result[0]["err"] + + def test_null_passthrough(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(items=None)], schema="items array") + result = df.select( + check_array_min_length(F.col("items"), 2).alias("err") + ).collect() + assert result[0]["err"] is None + + +class TestCheckMaxLength: + def test_within_limit(self, spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(items=["a", "b"])], schema="items array" + ) + result = df.select( + check_array_max_length(F.col("items"), 3).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_at_limit(self, spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(items=["a", "b"])], schema="items array" + ) + result = df.select( + check_array_max_length(F.col("items"), 2).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_exceeds_limit(self, spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(items=["a", "b", "c"])], schema="items array" + ) + result = df.select( + check_array_max_length(F.col("items"), 2).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "maximum length 2" in result[0]["err"] + + def test_null_passthrough(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(items=None)], schema="items array") + result = df.select( + check_array_max_length(F.col("items"), 2).alias("err") + ).collect() + assert result[0]["err"] is None + + +def test_check_require_any_of_satisfied(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(a=1, b=None)], schema="a int, b int") + result = df.select( + check_require_any_of([F.col("a"), F.col("b")], ["a", "b"]).alias("err") + ).collect() + assert result[0]["err"] is None + + +def test_check_require_any_of_all_null(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(a=None, b=None)], schema="a int, b int") + result = df.select( + check_require_any_of([F.col("a"), F.col("b")], ["a", "b"]).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "a" in result[0]["err"] + assert "b" in result[0]["err"] + + +class TestCheckRequireAnyTrue: + _NAMES = ["is_land", "is_territorial"] + + def _conds(self) -> list: + return [ + F.col("is_land") == F.lit(True), + F.col("is_territorial") == F.lit(True), + ] + + def test_one_condition_true(self, spark: SparkSession) -> None: + """At least one condition true -> no error.""" + df = spark.createDataFrame( + [Row(is_land=True, is_territorial=False)], + schema="is_land boolean, is_territorial boolean", + ) + result = df.select( + check_require_any_true(self._conds(), self._NAMES).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_all_conditions_false(self, spark: SparkSession) -> None: + """No condition true -> error naming the fields.""" + df = spark.createDataFrame( + [Row(is_land=False, is_territorial=False)], + schema="is_land boolean, is_territorial boolean", + ) + result = df.select( + check_require_any_true(self._conds(), self._NAMES).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "is_land" in result[0]["err"] + assert "is_territorial" in result[0]["err"] + + def test_all_conditions_null_is_violation(self, spark: SparkSession) -> None: + """Null fields -> conditions not true -> error. + + Mirrors Python's `None == True` -> `False`: a null column value + does not satisfy the condition, so an all-null row violates. + """ + df = spark.createDataFrame( + [Row(is_land=None, is_territorial=None)], + schema="is_land boolean, is_territorial boolean", + ) + result = df.select( + check_require_any_true(self._conds(), self._NAMES).alias("err") + ).collect() + assert result[0]["err"] is not None + + +class TestCheckRequireIf: + def test_required_present(self, spark: SparkSession) -> None: + """Target is present when condition is true -> no error.""" + df = spark.createDataFrame( + [("road", "primary")], schema="subtype string, road_class string" + ) + result = df.select( + check_require_if( + F.col("road_class"), + F.col("subtype").isin(["road", "rail"]), + "subtype in [road, rail]", + ).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_required_absent(self, spark: SparkSession) -> None: + """Target is null when condition is true -> error.""" + df = spark.createDataFrame( + [("road", None)], schema="subtype string, road_class string" + ) + result = df.select( + check_require_if( + F.col("road_class"), + F.col("subtype").isin(["road", "rail"]), + "subtype in [road, rail]", + ).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "required" in result[0]["err"] + + def test_condition_false_skips(self, spark: SparkSession) -> None: + """Target is null but condition is false -> no error.""" + df = spark.createDataFrame( + [("water", None)], schema="subtype string, road_class string" + ) + result = df.select( + check_require_if( + F.col("road_class"), + F.col("subtype").isin(["road", "rail"]), + "subtype in [road, rail]", + ).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_with_value_cols(self, spark: SparkSession) -> None: + """Error message includes actual discriminator value.""" + df = spark.createDataFrame( + [("road", None)], schema="subtype string, road_class string" + ) + result = df.select( + check_require_if( + F.col("road_class"), + F.col("subtype").isin(["road", "rail"]), + "subtype in [road, rail]", + F.col("subtype"), + ).alias("err") + ).collect() + assert "road" in result[0]["err"] + + +class TestCheckForbidIf: + def test_forbidden_absent(self, spark: SparkSession) -> None: + """Target is null when condition is true -> no error.""" + df = spark.createDataFrame( + [Row(subtype="country", parent=None)], + schema="subtype string, parent string", + ) + result = df.select( + check_forbid_if( + F.col("parent"), + F.col("subtype") == "country", + "subtype = country", + ).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_forbidden_present(self, spark: SparkSession) -> None: + """Target is present when condition is true -> error.""" + df = spark.createDataFrame([Row(subtype="country", parent="abc")]) + result = df.select( + check_forbid_if( + F.col("parent"), + F.col("subtype") == "country", + "subtype = country", + ).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "forbidden" in result[0]["err"] + + def test_condition_false_skips(self, spark: SparkSession) -> None: + """Target is present but condition is false -> no error.""" + df = spark.createDataFrame([Row(subtype="region", parent="abc")]) + result = df.select( + check_forbid_if( + F.col("parent"), + F.col("subtype") == "country", + "subtype = country", + ).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_with_value_cols(self, spark: SparkSession) -> None: + """Error message includes actual discriminator value.""" + df = spark.createDataFrame([Row(subtype="country", parent="abc")]) + result = df.select( + check_forbid_if( + F.col("parent"), + F.col("subtype") == "country", + "subtype = country", + F.col("subtype"), + ).alias("err") + ).collect() + assert "country" in result[0]["err"] + + +class TestCheckStringMinLength: + def test_valid_length(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="abc")]) + result = df.select( + check_string_min_length(F.col("val"), 1).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_empty_string_violation(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="")]) + result = df.select( + check_string_min_length(F.col("val"), 1).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "minimum length" in result[0]["err"] + + def test_null_passthrough(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=None)], schema="val string") + result = df.select( + check_string_min_length(F.col("val"), 1).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_exact_min_length(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="ab")]) + result = df.select( + check_string_min_length(F.col("val"), 2).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_below_min_length(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="a")]) + result = df.select( + check_string_min_length(F.col("val"), 2).alias("err") + ).collect() + assert result[0]["err"] is not None + + +class TestCheckStringMaxLength: + def test_valid_length(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="abc")]) + result = df.select( + check_string_max_length(F.col("val"), 5).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_above_max_length(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="abcdef")]) + result = df.select( + check_string_max_length(F.col("val"), 5).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "maximum length" in result[0]["err"] + + def test_null_passthrough(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=None)], schema="val string") + result = df.select( + check_string_max_length(F.col("val"), 5).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_exact_max_length(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="abcde")]) + result = df.select( + check_string_max_length(F.col("val"), 5).alias("err") + ).collect() + assert result[0]["err"] is None + + +class TestCheckRadioGroup: + def test_exactly_one_true(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(is_land=True, is_territorial=False)]) + result = df.select( + check_radio_group( + [F.col("is_land"), F.col("is_territorial")], + ["is_land", "is_territorial"], + ).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_none_true(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(is_land=False, is_territorial=False)]) + result = df.select( + check_radio_group( + [F.col("is_land"), F.col("is_territorial")], + ["is_land", "is_territorial"], + ).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "exactly one" in result[0]["err"] + assert "0" in result[0]["err"] + + def test_both_true(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(is_land=True, is_territorial=True)]) + result = df.select( + check_radio_group( + [F.col("is_land"), F.col("is_territorial")], + ["is_land", "is_territorial"], + ).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "2" in result[0]["err"] + + def test_null_treated_as_false(self, spark: SparkSession) -> None: + """Null booleans count as not-true (0 toward the count).""" + df = spark.createDataFrame( + [Row(is_land=True, is_territorial=None)], + schema="is_land boolean, is_territorial boolean", + ) + result = df.select( + check_radio_group( + [F.col("is_land"), F.col("is_territorial")], + ["is_land", "is_territorial"], + ).alias("err") + ).collect() + assert result[0]["err"] is None + + +class TestCheckGeometryType: + def test_point_matches(self, spark: SparkSession) -> None: + wkb_bytes = Point(0, 0).wkb + df = spark.createDataFrame( + [Row(geometry=bytearray(wkb_bytes))], schema="geometry binary" + ) + result = df.select( + check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_point_rejects_linestring(self, spark: SparkSession) -> None: + wkb_bytes = LineString([(0, 0), (1, 1)]).wkb + df = spark.createDataFrame( + [Row(geometry=bytearray(wkb_bytes))], schema="geometry binary" + ) + result = df.select( + check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "Point" in result[0]["err"] + + def test_multiple_allowed_types(self, spark: SparkSession) -> None: + wkb_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 0)]).wkb + wkb_multi = MultiPolygon([Polygon([(0, 0), (1, 0), (1, 1), (0, 0)])]).wkb + df = spark.createDataFrame( + [ + Row(geometry=bytearray(wkb_polygon)), + Row(geometry=bytearray(wkb_multi)), + ], + schema="geometry binary", + ) + result = df.select( + check_geometry_type( + F.col("geometry"), + GeometryType.POLYGON, + GeometryType.MULTI_POLYGON, + ).alias("err") + ).collect() + assert all(r["err"] is None for r in result) + + def test_multiple_allowed_rejects_wrong_type(self, spark: SparkSession) -> None: + wkb_point = Point(0, 0).wkb + df = spark.createDataFrame( + [Row(geometry=bytearray(wkb_point))], schema="geometry binary" + ) + result = df.select( + check_geometry_type( + F.col("geometry"), + GeometryType.POLYGON, + GeometryType.MULTI_POLYGON, + ).alias("err") + ).collect() + assert result[0]["err"] is not None + + def test_null_passthrough(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(geometry=None)], schema="geometry binary") + result = df.select( + check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_big_endian_wkb(self, spark: SparkSession) -> None: + """Verify BE byte order handling. + + Shapely writes LE by default. Construct BE WKB for a Point + manually: byte_order=0x00, type=0x00000001, x=0.0, y=0.0. + """ + be_point = struct.pack(">bIdd", 0, 1, 0.0, 0.0) + df = spark.createDataFrame( + [Row(geometry=bytearray(be_point))], schema="geometry binary" + ) + result = df.select( + check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_iso_wkb_z_point_accepted(self, spark: SparkSession) -> None: + """ISO WKB encodes Z by offsetting the type (PointZ=1001), shifting + the low byte to 0xE9. GeoParquet mandates ISO WKB, so 3D geometries + reach the check this way and must still validate by base type.""" + iso_point_z = struct.pack(" None: + iso_point_z_be = struct.pack(">bIddd", 0, 1001, 0.0, 0.0, 5.0) + df = spark.createDataFrame( + [Row(geometry=bytearray(iso_point_z_be))], schema="geometry binary" + ) + result = df.select( + check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_ewkb_z_point_accepted(self, spark: SparkSession) -> None: + """EWKB encodes Z as a high flag bit (0x80000001), leaving the low + byte at 0x01 -- shapely's `.wkb` default. Must keep validating.""" + ewkb_point_z = struct.pack(" None: + """shapely's native 3D WKB output validates as POINT.""" + df = spark.createDataFrame( + [Row(geometry=bytearray(Point(0, 0, 5).wkb))], schema="geometry binary" + ) + result = df.select( + check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_iso_wkb_z_wrong_type_rejected(self, spark: SparkSession) -> None: + """A 3D LineString (ISO 1002) is still rejected when POINT is expected + -- normalization strips the dimension offset, not the base type.""" + iso_linestring_z = struct.pack(" None: + """A non-null WKB blob too short to contain a type word is flagged as a violation.""" + truncated = bytearray(b"\x01") + df = spark.createDataFrame([Row(geometry=truncated)], schema="geometry binary") + result = df.select( + check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") + ).collect() + assert result[0]["err"] is not None + + @pytest.mark.parametrize("nbytes", [1, 2, 3, 4]) + def test_partial_header_wkb_flagged(self, spark: SparkSession, nbytes: int) -> None: + """A blob with a partial WKB header is flagged, even when conv() yields a non-null type. + + A little-endian order flag followed by a partial type word (2-4 bytes) + parses to a non-null but bogus base type -- e.g. `b"\\x01\\x01"` reads as + type 1, the Point code, and would silently validate as a Point. Only a + 0-1 byte blob makes conv() return NULL, so a length gate (not a null + check) is what closes the truncation hole. + """ + partial = bytearray(b"\x01" * nbytes) + df = spark.createDataFrame([Row(geometry=partial)], schema="geometry binary") + result = df.select( + check_geometry_type(F.col("geometry"), GeometryType.POINT).alias("err") + ).collect() + assert result[0]["err"] is not None + + +class TestCheckStripped: + def test_clean_string(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="hello world")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + def test_single_char(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="x")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + def test_leading_space(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=" hello")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + assert "whitespace" in result[0]["err"] + + def test_trailing_space(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="hello ")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + assert "whitespace" in result[0]["err"] + + def test_leading_tab(self, spark: SparkSession) -> None: + """Tab is Unicode whitespace -- must be caught (not just ASCII space).""" + df = spark.createDataFrame([Row(val="\thello")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + def test_trailing_newline(self, spark: SparkSession) -> None: + """Trailing newline requires \\z anchor -- $ matches before it in Java regex.""" + df = spark.createDataFrame([Row(val="hello\n")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + def test_null_passthrough(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=None)], schema="val string") + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + def test_empty_string(self, spark: SparkSession) -> None: + """Empty string has no leading/trailing whitespace -- passes.""" + df = spark.createDataFrame([Row(val="")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + def test_trailing_unit_separator(self, spark: SparkSession) -> None: + """U+001F (unit separator) -- Python strips it, Java \\S with (?U) does not.""" + df = spark.createDataFrame([Row(val="Main St \x1f")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + def test_leading_file_separator(self, spark: SparkSession) -> None: + """U+001C (file separator) -- C0 control char Python treats as whitespace.""" + df = spark.createDataFrame([Row(val="\x1chello")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + def test_trailing_soh(self, spark: SparkSession) -> None: + """U+0001 (SOH) -- C0 control char that even Python's strip() misses.""" + df = spark.createDataFrame([Row(val="hello\x01")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + def test_trailing_del(self, spark: SparkSession) -> None: + """U+007F (DEL) -- control char outside C0 range.""" + df = spark.createDataFrame([Row(val="hello\x7f")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + def test_trailing_c1_control(self, spark: SparkSession) -> None: + """U+009F (APC) -- C1 control char.""" + df = spark.createDataFrame([Row(val="hello\x9f")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + def test_control_char_in_middle_passes(self, spark: SparkSession) -> None: + """Control chars in the middle of a string are not a stripped concern.""" + df = spark.createDataFrame([Row(val="hel\x1flo")]) + result = df.select(check_stripped(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +class TestCheckJsonPointer: + def test_valid_pointer(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="/properties/name")]) + result = df.select(check_json_pointer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + def test_root_pointer(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="/")]) + result = df.select(check_json_pointer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + def test_empty_string_valid(self, spark: SparkSession) -> None: + """Empty string is valid per RFC 6901 (references whole document).""" + df = spark.createDataFrame([Row(val="")]) + result = df.select(check_json_pointer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + def test_missing_leading_slash(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="properties/name")]) + result = df.select(check_json_pointer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + assert "JSON pointer" in result[0]["err"] + assert "properties/name" in result[0]["err"] + + def test_null_passthrough(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=None)], schema="val string") + result = df.select(check_json_pointer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +class TestCheckLinearRangeLength: + def test_valid_length(self, spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(between=[0.0, 1.0])], schema="between array" + ) + result = df.select( + check_linear_range_length(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_wrong_length_one(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(between=[0.5])], schema="between array") + result = df.select( + check_linear_range_length(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "2 elements" in result[0]["err"] + + def test_wrong_length_three(self, spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(between=[0.0, 0.5, 1.0])], schema="between array" + ) + result = df.select( + check_linear_range_length(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "2 elements" in result[0]["err"] + + def test_empty_array(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(between=[])], schema="between array") + result = df.select( + check_linear_range_length(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "2 elements" in result[0]["err"] + + def test_null_passthrough(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(between=None)], schema="between array") + result = df.select( + check_linear_range_length(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is None + + +class TestCheckLinearRangeBounds: + def test_valid_bounds(self, spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(between=[0.2, 0.8])], schema="between array" + ) + result = df.select( + check_linear_range_bounds(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_value_below_zero(self, spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(between=[-0.1, 0.5])], schema="between array" + ) + result = df.select( + check_linear_range_bounds(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "[0.0, 1.0]" in result[0]["err"] + + def test_value_above_one(self, spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(between=[0.0, 1.1])], schema="between array" + ) + result = df.select( + check_linear_range_bounds(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "[0.0, 1.0]" in result[0]["err"] + + def test_wrong_length_passthrough(self, spark: SparkSession) -> None: + """Wrong-length arrays are not this function's concern.""" + df = spark.createDataFrame([Row(between=[0.5])], schema="between array") + result = df.select( + check_linear_range_bounds(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_null_passthrough(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(between=None)], schema="between array") + result = df.select( + check_linear_range_bounds(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is None + + +class TestCheckLinearRangeOrder: + def test_valid_order(self, spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(between=[0.2, 0.8])], schema="between array" + ) + result = df.select( + check_linear_range_order(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_start_equals_end(self, spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(between=[0.5, 0.5])], schema="between array" + ) + result = df.select( + check_linear_range_order(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "start must be < end" in result[0]["err"] + + def test_start_after_end(self, spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(between=[0.8, 0.2])], schema="between array" + ) + result = df.select( + check_linear_range_order(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is not None + assert "start must be < end" in result[0]["err"] + + def test_wrong_length_passthrough(self, spark: SparkSession) -> None: + """Wrong-length arrays are not this function's concern.""" + df = spark.createDataFrame([Row(between=[0.5])], schema="between array") + result = df.select( + check_linear_range_order(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_null_passthrough(self, spark: SparkSession) -> None: + df = spark.createDataFrame([Row(between=None)], schema="between array") + result = df.select( + check_linear_range_order(F.col("between")).alias("err") + ).collect() + assert result[0]["err"] is None + + +def test_check_required_null_is_error(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=None)], schema="val string") + result = df.select(check_required(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + assert "missing" in result[0]["err"] + + +def test_check_required_non_null_passes(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="hello")]) + result = df.select(check_required(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_required_composes_with_enum(spark: SparkSession) -> None: + """check_required + check_enum via F.coalesce catches both null and invalid.""" + df = spark.createDataFrame([Row(val=None)], schema="val string") + expr = F.coalesce( + check_required(F.col("val")), + check_enum(F.col("val"), ["a", "b"]), + ) + result = df.select(expr.alias("err")).collect() + assert result[0]["err"] is not None + assert "missing" in result[0]["err"] + + +_COUNTRY_CODE_PATTERN = r"^[A-Z]{2}\z" +_COUNTRY_CODE_LABEL = "ISO 3166-1 alpha-2 country code" + + +class TestCheckCountryCodeViaPattern: + """Country code validation through check_pattern with label.""" + + def test_valid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("US",)], ["v"]) + result = df.select( + check_pattern( + F.col("v"), _COUNTRY_CODE_PATTERN, label=_COUNTRY_CODE_LABEL + ).alias("e") + ) + assert result.collect()[0]["e"] is None + + def test_lowercase_invalid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("us",)], ["v"]) + result = df.select( + check_pattern( + F.col("v"), _COUNTRY_CODE_PATTERN, label=_COUNTRY_CODE_LABEL + ).alias("e") + ) + err = result.collect()[0]["e"] + assert f"invalid {_COUNTRY_CODE_LABEL}" in err + assert "us" in err + + def test_three_chars_invalid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("USA",)], ["v"]) + result = df.select( + check_pattern( + F.col("v"), _COUNTRY_CODE_PATTERN, label=_COUNTRY_CODE_LABEL + ).alias("e") + ) + assert result.collect()[0]["e"] is not None + + def test_null_passes(self, spark: SparkSession) -> None: + df = spark.createDataFrame([(None,)], schema="v string") + result = df.select( + check_pattern( + F.col("v"), _COUNTRY_CODE_PATTERN, label=_COUNTRY_CODE_LABEL + ).alias("e") + ) + assert result.collect()[0]["e"] is None + + +_REGION_CODE_PATTERN = r"^[A-Z]{2}-[A-Z0-9]{1,3}\z" +_REGION_CODE_LABEL = "ISO 3166-2 subdivision code" + + +class TestCheckRegionCodeViaPattern: + """Region code validation through check_pattern with label.""" + + def test_valid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("US-NY",)], ["v"]) + result = df.select( + check_pattern( + F.col("v"), _REGION_CODE_PATTERN, label=_REGION_CODE_LABEL + ).alias("e") + ) + assert result.collect()[0]["e"] is None + + def test_valid_numeric(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("CN-11",)], ["v"]) + result = df.select( + check_pattern( + F.col("v"), _REGION_CODE_PATTERN, label=_REGION_CODE_LABEL + ).alias("e") + ) + assert result.collect()[0]["e"] is None + + def test_no_dash_invalid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("USNY",)], ["v"]) + result = df.select( + check_pattern( + F.col("v"), _REGION_CODE_PATTERN, label=_REGION_CODE_LABEL + ).alias("e") + ) + err = result.collect()[0]["e"] + assert f"invalid {_REGION_CODE_LABEL}" in err + assert "USNY" in err + + def test_null_passes(self, spark: SparkSession) -> None: + df = spark.createDataFrame([(None,)], schema="v string") + result = df.select( + check_pattern( + F.col("v"), _REGION_CODE_PATTERN, label=_REGION_CODE_LABEL + ).alias("e") + ) + assert result.collect()[0]["e"] is None + + +_SNAKE_CASE_PATTERN = r"^[a-z0-9]+(_[a-z0-9]+)*\z" +_SNAKE_CASE_LABEL = "Category in snake_case format" + + +class TestCheckSnakeCaseViaPattern: + """Snake_case validation through check_pattern with label.""" + + def test_valid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("hello_world",)], ["v"]) + result = df.select( + check_pattern( + F.col("v"), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL + ).alias("e") + ) + assert result.collect()[0]["e"] is None + + def test_single_word(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("hello",)], ["v"]) + result = df.select( + check_pattern( + F.col("v"), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL + ).alias("e") + ) + assert result.collect()[0]["e"] is None + + def test_with_numbers(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("hello_123",)], ["v"]) + result = df.select( + check_pattern( + F.col("v"), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL + ).alias("e") + ) + assert result.collect()[0]["e"] is None + + def test_uppercase_invalid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("Hello_World",)], ["v"]) + result = df.select( + check_pattern( + F.col("v"), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL + ).alias("e") + ) + err = result.collect()[0]["e"] + assert f"invalid {_SNAKE_CASE_LABEL}" in err + + def test_spaces_invalid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("hello world",)], ["v"]) + result = df.select( + check_pattern( + F.col("v"), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL + ).alias("e") + ) + assert result.collect()[0]["e"] is not None + + def test_null_passes(self, spark: SparkSession) -> None: + df = spark.createDataFrame([(None,)], schema="v string") + result = df.select( + check_pattern( + F.col("v"), _SNAKE_CASE_PATTERN, label=_SNAKE_CASE_LABEL + ).alias("e") + ) + assert result.collect()[0]["e"] is None + + +def test_check_url_format_http_valid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="http://example.com")]) + result = df.select(check_url_format(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_url_format_https_valid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="https://example.com/path?q=1")]) + result = df.select(check_url_format(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_url_format_uppercase_scheme_valid(spark: SparkSession) -> None: + """Pydantic HttpUrl lowercases the scheme, so HTTP:// is accepted.""" + df = spark.createDataFrame([Row(val="HTTP://example.com")]) + result = df.select(check_url_format(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_url_format_mixed_case_scheme_valid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="HtTpS://example.com/path")]) + result = df.select(check_url_format(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_url_format_no_scheme_invalid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="example.com")]) + result = df.select(check_url_format(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_url_format_ftp_scheme_invalid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="ftp://example.com")]) + result = df.select(check_url_format(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_url_format_null_passes(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=None)], schema="val string") + result = df.select(check_url_format(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_url_length_exceeds_2083_chars_invalid(spark: SparkSession) -> None: + long_url = "https://example.com/" + "a" * 2064 # 2084 chars + df = spark.createDataFrame([Row(val=long_url)]) + result = df.select(check_url_length(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_url_length_exactly_2083_chars_valid(spark: SparkSession) -> None: + url = "https://example.com/" + "a" * 2063 # 2083 chars + df = spark.createDataFrame([Row(val=url)]) + result = df.select(check_url_length(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_url_length_null_passes(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=None)], schema="val string") + result = df.select(check_url_length(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_email_valid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="user@example.com")]) + result = df.select(check_email(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_email_no_at_invalid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="userexample.com")]) + result = df.select(check_email(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_email_no_domain_invalid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="user@")]) + result = df.select(check_email(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_email_spaces_invalid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="user @example.com")]) + result = df.select(check_email(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_email_null_passes(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=None)], schema="val string") + result = df.select(check_email(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_email_trailing_period_invalid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="user@example.com.")]) + result = df.select(check_email(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_email_leading_period_invalid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=".user@example.com")]) + result = df.select(check_email(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_email_period_before_at_invalid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="user.@example.com")]) + result = df.select(check_email(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_email_period_after_at_invalid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="user@.example.com")]) + result = df.select(check_email(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_email_double_period_domain_invalid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="user@example..com")]) + result = df.select(check_email(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_email_dotted_local_valid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="user.name@example.com")]) + result = df.select(check_email(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_email_subdomain_valid(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val="user@mail.example.co.uk")]) + result = df.select(check_email(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +_PHONE_PATTERN = r"^\+\d{1,3}[\s\-\(\)0-9]+\z" +_PHONE_LABEL = "International phone number (+ followed by country code and number)" + + +class TestCheckPhoneViaPattern: + """Phone number validation through check_pattern with label.""" + + def test_valid_us(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("+1 555-555-5555",)], ["v"]) + result = df.select( + check_pattern(F.col("v"), _PHONE_PATTERN, label=_PHONE_LABEL).alias("e") + ) + assert result.collect()[0]["e"] is None + + def test_valid_international(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("+44 20 7946 0958",)], ["v"]) + result = df.select( + check_pattern(F.col("v"), _PHONE_PATTERN, label=_PHONE_LABEL).alias("e") + ) + assert result.collect()[0]["e"] is None + + def test_no_plus_invalid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("555-555-5555",)], ["v"]) + result = df.select( + check_pattern(F.col("v"), _PHONE_PATTERN, label=_PHONE_LABEL).alias("e") + ) + err = result.collect()[0]["e"] + assert f"invalid {_PHONE_LABEL}" in err + + def test_letters_invalid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("+1 abc-defg",)], ["v"]) + result = df.select( + check_pattern(F.col("v"), _PHONE_PATTERN, label=_PHONE_LABEL).alias("e") + ) + assert result.collect()[0]["e"] is not None + + def test_null_passes(self, spark: SparkSession) -> None: + df = spark.createDataFrame([(None,)], schema="v string") + result = df.select( + check_pattern(F.col("v"), _PHONE_PATTERN, label=_PHONE_LABEL).alias("e") + ) + assert result.collect()[0]["e"] is None + + +_WIKIDATA_PATTERN = r"^Q\d+\z" +_WIKIDATA_LABEL = "Wikidata identifier (Q followed by digits)" + + +class TestCheckWikidataIdViaPattern: + """Wikidata ID validation through check_pattern with label.""" + + def test_valid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("Q42",)], ["v"]) + result = df.select( + check_pattern(F.col("v"), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL).alias( + "e" + ) + ) + assert result.collect()[0]["e"] is None + + def test_large_number(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("Q123456789",)], ["v"]) + result = df.select( + check_pattern(F.col("v"), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL).alias( + "e" + ) + ) + assert result.collect()[0]["e"] is None + + def test_lowercase_q_invalid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("q42",)], ["v"]) + result = df.select( + check_pattern(F.col("v"), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL).alias( + "e" + ) + ) + err = result.collect()[0]["e"] + assert f"invalid {_WIKIDATA_LABEL}" in err + + def test_no_digits_invalid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("Q",)], ["v"]) + result = df.select( + check_pattern(F.col("v"), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL).alias( + "e" + ) + ) + assert result.collect()[0]["e"] is not None + + def test_p_prefix_invalid(self, spark: SparkSession) -> None: + df = spark.createDataFrame([("P42",)], ["v"]) + result = df.select( + check_pattern(F.col("v"), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL).alias( + "e" + ) + ) + assert result.collect()[0]["e"] is not None + + def test_null_passes(self, spark: SparkSession) -> None: + df = spark.createDataFrame([(None,)], schema="v string") + result = df.select( + check_pattern(F.col("v"), _WIKIDATA_PATTERN, label=_WIKIDATA_LABEL).alias( + "e" + ) + ) + assert result.collect()[0]["e"] is None + + +class TestCheckMinFieldsSet: + def test_meets_threshold(self, spark: SparkSession) -> None: + """Count at threshold -> no error.""" + df = spark.createDataFrame( + [Row(a=1, b=2, c=None)], schema="a int, b int, c int" + ) + result = df.select( + check_min_fields_set( + [F.col("a"), F.col("b"), F.col("c")], + ["a", "b", "c"], + 2, + ).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_exceeds_threshold(self, spark: SparkSession) -> None: + """Count above threshold -> no error.""" + df = spark.createDataFrame([Row(a=1, b=2, c=3)], schema="a int, b int, c int") + result = df.select( + check_min_fields_set( + [F.col("a"), F.col("b"), F.col("c")], + ["a", "b", "c"], + 2, + ).alias("err") + ).collect() + assert result[0]["err"] is None + + def test_below_threshold(self, spark: SparkSession) -> None: + """Count below threshold -> error with field names and actual count.""" + df = spark.createDataFrame( + [Row(a=1, b=None, c=None)], schema="a int, b int, c int" + ) + result = df.select( + check_min_fields_set( + [F.col("a"), F.col("b"), F.col("c")], + ["a", "b", "c"], + 2, + ).alias("err") + ).collect() + err = result[0]["err"] + assert err is not None + assert "at least 2" in err + assert "a, b, c" in err + assert "1" in err + + def test_all_null_below_threshold(self, spark: SparkSession) -> None: + """All null -> error showing 0 non-null.""" + df = spark.createDataFrame([Row(a=None, b=None)], schema="a int, b int") + result = df.select( + check_min_fields_set( + [F.col("a"), F.col("b")], + ["a", "b"], + 1, + ).alias("err") + ).collect() + err = result[0]["err"] + assert err is not None + assert "0" in err + + def test_error_message_format(self, spark: SparkSession) -> None: + """Error message matches expected format exactly.""" + df = spark.createDataFrame([Row(x=None, y=None)], schema="x int, y int") + result = df.select( + check_min_fields_set( + [F.col("x"), F.col("y")], + ["x", "y"], + 1, + ).alias("err") + ).collect() + err = result[0]["err"] + assert err == "at least 1 of x, y required, got 0 non-null" + + +_BBOX_SCHEMA = StructType( + [ + StructField( + "bbox", + StructType( + [ + StructField("xmin", DoubleType(), True), + StructField("xmax", DoubleType(), True), + StructField("ymin", DoubleType(), True), + StructField("ymax", DoubleType(), True), + ] + ), + True, + ), + ] +) + + +def test_check_bbox_completeness_valid(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(bbox=Row(xmin=0.0, xmax=1.0, ymin=0.0, ymax=1.0))], + schema=_BBOX_SCHEMA, + ) + result = df.select(check_bbox_completeness(F.col("bbox")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_bbox_completeness_null_bbox_passes(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(bbox=None)], schema=_BBOX_SCHEMA) + result = df.select(check_bbox_completeness(F.col("bbox")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_bbox_completeness_null_subfield_fails(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(bbox=Row(xmin=None, xmax=1.0, ymin=0.0, ymax=1.0))], + schema=_BBOX_SCHEMA, + ) + result = df.select(check_bbox_completeness(F.col("bbox")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_bbox_lat_ordering_valid(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(bbox=Row(xmin=0.0, xmax=1.0, ymin=-10.0, ymax=10.0))], + schema=_BBOX_SCHEMA, + ) + result = df.select(check_bbox_lat_ordering(F.col("bbox")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_bbox_lat_ordering_equal_valid(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(bbox=Row(xmin=0.0, xmax=1.0, ymin=5.0, ymax=5.0))], + schema=_BBOX_SCHEMA, + ) + result = df.select(check_bbox_lat_ordering(F.col("bbox")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_bbox_lat_ordering_inverted_fails(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(bbox=Row(xmin=0.0, xmax=1.0, ymin=10.0, ymax=-10.0))], + schema=_BBOX_SCHEMA, + ) + result = df.select(check_bbox_lat_ordering(F.col("bbox")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_bbox_lat_ordering_null_bbox_passes(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(bbox=None)], schema=_BBOX_SCHEMA) + result = df.select(check_bbox_lat_ordering(F.col("bbox")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_bbox_lat_range_valid(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(bbox=Row(xmin=0.0, xmax=1.0, ymin=-90.0, ymax=90.0))], + schema=_BBOX_SCHEMA, + ) + result = df.select(check_bbox_lat_range(F.col("bbox")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_bbox_lat_range_ymin_below_fails(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(bbox=Row(xmin=0.0, xmax=1.0, ymin=-91.0, ymax=1.0))], + schema=_BBOX_SCHEMA, + ) + result = df.select(check_bbox_lat_range(F.col("bbox")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_bbox_lat_range_ymax_above_fails(spark: SparkSession) -> None: + df = spark.createDataFrame( + [Row(bbox=Row(xmin=0.0, xmax=1.0, ymin=0.0, ymax=91.0))], + schema=_BBOX_SCHEMA, + ) + result = df.select(check_bbox_lat_range(F.col("bbox")).alias("err")).collect() + assert result[0]["err"] is not None + + +def test_check_bbox_lat_range_null_bbox_passes(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(bbox=None)], schema=_BBOX_SCHEMA) + result = df.select(check_bbox_lat_range(F.col("bbox")).alias("err")).collect() + assert result[0]["err"] is None diff --git a/packages/overture-schema-pyspark/tests/expressions/test_schema_check.py b/packages/overture-schema-pyspark/tests/expressions/test_schema_check.py new file mode 100644 index 000000000..ddd756a00 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/expressions/test_schema_check.py @@ -0,0 +1,298 @@ +"""Tests for schema comparison.""" + +from overture.schema.pyspark.schema_check import ( + SchemaMismatch, + compare_schemas, +) +from pyspark.sql.types import ( + ArrayType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + + +class TestIdenticalSchemas: + def test_empty_schemas(self) -> None: + assert compare_schemas(StructType(), StructType()) == [] + + def test_flat_schema(self) -> None: + schema = StructType( + [ + StructField("id", StringType(), True), + StructField("version", IntegerType(), True), + ] + ) + assert compare_schemas(schema, schema) == [] + + def test_nested_struct(self) -> None: + schema = StructType( + [ + StructField( + "bbox", + StructType( + [ + StructField("xmin", DoubleType(), True), + ] + ), + True, + ), + ] + ) + assert compare_schemas(schema, schema) == [] + + def test_array_of_structs(self) -> None: + schema = StructType( + [ + StructField( + "items", + ArrayType( + StructType( + [ + StructField("name", StringType(), True), + ] + ) + ), + True, + ), + ] + ) + assert compare_schemas(schema, schema) == [] + + +class TestMissingFields: + def test_missing_in_actual(self) -> None: + actual = StructType([StructField("id", StringType(), True)]) + expected = StructType( + [ + StructField("id", StringType(), True), + StructField("version", IntegerType(), True), + ] + ) + result = compare_schemas(actual, expected) + assert result == [SchemaMismatch("version", "missing", "IntegerType")] + + def test_extra_in_actual(self) -> None: + actual = StructType( + [ + StructField("id", StringType(), True), + StructField("extra", StringType(), True), + ] + ) + expected = StructType([StructField("id", StringType(), True)]) + result = compare_schemas(actual, expected) + assert result == [SchemaMismatch("extra", "StringType", "missing")] + + +class TestTypeMismatches: + def test_top_level_type_mismatch(self) -> None: + actual = StructType([StructField("version", StringType(), True)]) + expected = StructType([StructField("version", IntegerType(), True)]) + result = compare_schemas(actual, expected) + assert result == [SchemaMismatch("version", "StringType", "IntegerType")] + + def test_nested_struct_mismatch(self) -> None: + actual = StructType( + [ + StructField( + "bbox", + StructType( + [ + StructField("xmin", IntegerType(), True), + ] + ), + True, + ), + ] + ) + expected = StructType( + [ + StructField( + "bbox", + StructType( + [ + StructField("xmin", DoubleType(), True), + ] + ), + True, + ), + ] + ) + result = compare_schemas(actual, expected) + assert result == [SchemaMismatch("bbox.xmin", "IntegerType", "DoubleType")] + + def test_array_element_type_mismatch(self) -> None: + actual = StructType( + [ + StructField("tags", ArrayType(IntegerType()), True), + ] + ) + expected = StructType( + [ + StructField("tags", ArrayType(StringType()), True), + ] + ) + result = compare_schemas(actual, expected) + assert result == [SchemaMismatch("tags[]", "IntegerType", "StringType")] + + def test_array_struct_field_mismatch(self) -> None: + actual = StructType( + [ + StructField( + "items", + ArrayType( + StructType( + [ + StructField("name", IntegerType(), True), + ] + ) + ), + True, + ), + ] + ) + expected = StructType( + [ + StructField( + "items", + ArrayType( + StructType( + [ + StructField("name", StringType(), True), + ] + ) + ), + True, + ), + ] + ) + result = compare_schemas(actual, expected) + assert result == [SchemaMismatch("items[].name", "IntegerType", "StringType")] + + def test_map_key_type_mismatch(self) -> None: + actual = StructType( + [ + StructField("tags", MapType(IntegerType(), StringType()), True), + ] + ) + expected = StructType( + [ + StructField("tags", MapType(StringType(), StringType()), True), + ] + ) + result = compare_schemas(actual, expected) + assert result == [SchemaMismatch("tags{key}", "IntegerType", "StringType")] + + def test_map_value_type_mismatch(self) -> None: + actual = StructType( + [ + StructField("tags", MapType(StringType(), IntegerType()), True), + ] + ) + expected = StructType( + [ + StructField("tags", MapType(StringType(), StringType()), True), + ] + ) + result = compare_schemas(actual, expected) + assert result == [SchemaMismatch("tags{value}", "IntegerType", "StringType")] + + +class TestFieldOrdering: + def test_different_order_is_ok(self) -> None: + actual = StructType( + [ + StructField("b", StringType(), True), + StructField("a", IntegerType(), True), + ] + ) + expected = StructType( + [ + StructField("a", IntegerType(), True), + StructField("b", StringType(), True), + ] + ) + assert compare_schemas(actual, expected) == [] + + +class TestMultipleMismatches: + def test_missing_and_extra_and_wrong_type(self) -> None: + actual = StructType( + [ + StructField("id", IntegerType(), True), + StructField("extra", StringType(), True), + ] + ) + expected = StructType( + [ + StructField("id", StringType(), True), + StructField("version", IntegerType(), True), + ] + ) + result = compare_schemas(actual, expected) + assert SchemaMismatch("id", "IntegerType", "StringType") in result + assert SchemaMismatch("extra", "StringType", "missing") in result + assert SchemaMismatch("version", "missing", "IntegerType") in result + + +class TestKindMismatch: + def test_struct_vs_primitive(self) -> None: + actual = StructType([StructField("x", StringType(), True)]) + expected = StructType( + [ + StructField( + "x", + StructType( + [ + StructField("y", StringType(), True), + ] + ), + True, + ), + ] + ) + result = compare_schemas(actual, expected) + assert result == [SchemaMismatch("x", "StringType", "StructType")] + + def test_array_vs_primitive(self) -> None: + actual = StructType([StructField("x", StringType(), True)]) + expected = StructType( + [ + StructField("x", ArrayType(StringType()), True), + ] + ) + result = compare_schemas(actual, expected) + assert result == [SchemaMismatch("x", "StringType", "ArrayType")] + + +class TestSchemaMismatchRoot: + """`root` strips the step markers `_compare` embeds in `path`. + + The top-level column a mismatch belongs to is everything before the + first struct (`.`), array (`[]`), or map (`{key}`/`{value}`) step, so + it matches the column-granular `Check.read_columns`. + """ + + def test_top_level(self) -> None: + assert SchemaMismatch("theme", "missing", "StringType").root == "theme" + + def test_struct_field(self) -> None: + assert SchemaMismatch("bbox.xmin", "missing", "DoubleType").root == "bbox" + + def test_array_element_field(self) -> None: + assert ( + SchemaMismatch("sources[].confidence", "missing", "DoubleType").root + == "sources" + ) + + def test_array_element(self) -> None: + assert SchemaMismatch("tags[]", "IntegerType", "StringType").root == "tags" + + def test_map_key(self) -> None: + assert SchemaMismatch("tags{key}", "IntegerType", "StringType").root == "tags" + + def test_map_value(self) -> None: + assert SchemaMismatch("tags{value}", "IntegerType", "StringType").root == "tags" diff --git a/packages/overture-schema-pyspark/tests/generated/__init__.py b/packages/overture-schema-pyspark/tests/generated/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/generated/overture/__init__.py b/packages/overture-schema-pyspark/tests/generated/overture/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/__init__.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/addresses/__init__.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/addresses/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/addresses/test_address.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/addresses/test_address.py new file mode 100644 index 000000000..5cf2e5559 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/addresses/test_address.py @@ -0,0 +1,524 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for address.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.addresses.address import ( + ADDRESS_SCHEMA, + address_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import mutate_unique_items +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "9b51bb94-b26f-5f88-ad00-affc1e8f1935", + "geometry": "POINT (0 0)", + "theme": "addresses", + "type": "address", + "version": 0, + "country": "US", +} + + +BASE_ROW_POPULATED: dict = { + "id": "9b51bb94-b26f-5f88-ad00-affc1e8f1935", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "POINT (0 0)", + "theme": "addresses", + "type": "address", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "address_levels": [{"value": "a"}], + "country": "US", + "number": "a", + "postal_city": "a", + "postcode": "a", + "street": "a", + "unit": "a", +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="address::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="address::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="address::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="address::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="address::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="address::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="address::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="address::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "LINESTRING (0 0, 1 1)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="address::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="address::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="address::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="address::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="address::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="address::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="address::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="address::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="address::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="address::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="address::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="address::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="address::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="address::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="address::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="address::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="address::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="address::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="address::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="address::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="address::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="address::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="address::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="address::address_levels_min_length:array_min_length", + scaffold={"address_levels": [{}]}, + mutate=set_at_path("address_levels", []), + expected_field="address_levels_min_length", + expected_check="array_min_length", + ), + Scenario( + id="address::address_levels_max_length:array_max_length", + scaffold={"address_levels": [{}]}, + mutate=set_at_path("address_levels", [{}, {}, {}, {}, {}, {}]), + expected_field="address_levels_max_length", + expected_check="array_max_length", + ), + Scenario( + id="address::address_levels[].value:string_min_length", + scaffold={"address_levels": [{"value": "a"}]}, + mutate=set_at_path("address_levels[].value", ""), + expected_field="address_levels[].value", + expected_check="string_min_length", + ), + Scenario( + id="address::address_levels[].value:stripped", + scaffold={"address_levels": [{"value": "a"}]}, + mutate=set_at_path("address_levels[].value", " has spaces "), + expected_field="address_levels[].value", + expected_check="stripped", + ), + Scenario( + id="address::country:required", + scaffold={}, + mutate=set_at_path("country", None), + expected_field="country", + expected_check="required", + ), + Scenario( + id="address::country:country_code_alpha2", + scaffold={}, + mutate=set_at_path("country", "99"), + expected_field="country", + expected_check="country_code_alpha2", + ), + Scenario( + id="address::number:string_min_length", + scaffold={"number": "a"}, + mutate=set_at_path("number", ""), + expected_field="number", + expected_check="string_min_length", + ), + Scenario( + id="address::number:stripped", + scaffold={"number": "a"}, + mutate=set_at_path("number", " has spaces "), + expected_field="number", + expected_check="stripped", + ), + Scenario( + id="address::postal_city:string_min_length", + scaffold={"postal_city": "a"}, + mutate=set_at_path("postal_city", ""), + expected_field="postal_city", + expected_check="string_min_length", + ), + Scenario( + id="address::postal_city:stripped", + scaffold={"postal_city": "a"}, + mutate=set_at_path("postal_city", " has spaces "), + expected_field="postal_city", + expected_check="stripped", + ), + Scenario( + id="address::postcode:string_min_length", + scaffold={"postcode": "a"}, + mutate=set_at_path("postcode", ""), + expected_field="postcode", + expected_check="string_min_length", + ), + Scenario( + id="address::postcode:stripped", + scaffold={"postcode": "a"}, + mutate=set_at_path("postcode", " has spaces "), + expected_field="postcode", + expected_check="stripped", + ), + Scenario( + id="address::street:string_min_length", + scaffold={"street": "a"}, + mutate=set_at_path("street", ""), + expected_field="street", + expected_check="string_min_length", + ), + Scenario( + id="address::street:stripped", + scaffold={"street": "a"}, + mutate=set_at_path("street", " has spaces "), + expected_field="street", + expected_check="stripped", + ), + Scenario( + id="address::unit:string_min_length", + scaffold={"unit": "a"}, + mutate=set_at_path("unit", ""), + expected_field="unit", + expected_check="string_min_length", + ), + Scenario( + id="address::unit:stripped", + scaffold={"unit": "a"}, + mutate=set_at_path("unit", " has spaces "), + expected_field="unit", + expected_check="stripped", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return address_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + ADDRESS_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="address", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + ADDRESS_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="address", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("address::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("address::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/annex/__init__.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/annex/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/annex/test_sources.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/annex/test_sources.py new file mode 100644 index 000000000..b46940e9e --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/annex/test_sources.py @@ -0,0 +1,1042 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for sources.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.annex.sources import ( + SOURCES_SCHEMA, + sources_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import mutate_map_key, mutate_map_value +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ], + "license_priority": {"ODbL-1.0": 0}, +} + + +BASE_ROW_POPULATED: dict = { + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "inception_date": "2024-01-01", + "url": "https://example.com/", + "url_archived": "https://example.com/", + "data_download_url": ["https://example.com/"], + "countries": ["US"], + "coverage_description": "", + "data_layer_name": "", + "oa_path": [""], + "address_levels": [""], + "file_format": "", + "update_frequency": "", + "build_source": "OpenAddresses", + "update_type": "continuous", + "update_schedule": [""], + "known_issues": "", + "notes": "", + "requires_attribution": "", + } + ], + "license_priority": {"ODbL-1.0": 0}, +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="sources::datasets:required", + scaffold={}, + mutate=set_at_path("datasets", None), + expected_field="datasets", + expected_check="required", + ), + Scenario( + id="sources::datasets[].source_name:required", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].source_name", None), + expected_field="datasets[].source_name", + expected_check="required", + ), + Scenario( + id="sources::datasets[].source_dataset_name:required", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].source_dataset_name", None), + expected_field="datasets[].source_dataset_name", + expected_check="required", + ), + Scenario( + id="sources::datasets[].data_url:required", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].data_url", None), + expected_field="datasets[].data_url", + expected_check="required", + ), + Scenario( + id="sources::datasets[].data_url:url_format", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].data_url", "not-a-url"), + expected_field="datasets[].data_url", + expected_check="url_format", + valid_scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + ), + Scenario( + id="sources::datasets[].data_url:url_length", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path( + "datasets[].data_url", + "https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + ), + expected_field="datasets[].data_url", + expected_check="url_length", + valid_scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + ), + Scenario( + id="sources::datasets[].data_url_archived:required", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].data_url_archived", None), + expected_field="datasets[].data_url_archived", + expected_check="required", + ), + Scenario( + id="sources::datasets[].data_url_archived:url_format", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].data_url_archived", "not-a-url"), + expected_field="datasets[].data_url_archived", + expected_check="url_format", + valid_scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + ), + Scenario( + id="sources::datasets[].data_url_archived:url_length", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path( + "datasets[].data_url_archived", + "https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + ), + expected_field="datasets[].data_url_archived", + expected_check="url_length", + valid_scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + ), + Scenario( + id="sources::datasets[].license_url:required", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].license_url", None), + expected_field="datasets[].license_url", + expected_check="required", + ), + Scenario( + id="sources::datasets[].license_url:url_format", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].license_url", "not-a-url"), + expected_field="datasets[].license_url", + expected_check="url_format", + valid_scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + ), + Scenario( + id="sources::datasets[].license_url:url_length", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path( + "datasets[].license_url", + "https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + ), + expected_field="datasets[].license_url", + expected_check="url_length", + valid_scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + ), + Scenario( + id="sources::datasets[].license_url_archived:required", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].license_url_archived", None), + expected_field="datasets[].license_url_archived", + expected_check="required", + ), + Scenario( + id="sources::datasets[].license_url_archived:url_format", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].license_url_archived", "not-a-url"), + expected_field="datasets[].license_url_archived", + expected_check="url_format", + valid_scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + ), + Scenario( + id="sources::datasets[].license_url_archived:url_length", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path( + "datasets[].license_url_archived", + "https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + ), + expected_field="datasets[].license_url_archived", + expected_check="url_length", + valid_scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + ), + Scenario( + id="sources::datasets[].license_type:required", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].license_type", None), + expected_field="datasets[].license_type", + expected_check="required", + ), + Scenario( + id="sources::datasets[].license_text:required", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].license_text", None), + expected_field="datasets[].license_text", + expected_check="required", + ), + Scenario( + id="sources::datasets[].license_attribution:required", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].license_attribution", None), + expected_field="datasets[].license_attribution", + expected_check="required", + ), + Scenario( + id="sources::datasets[].coverage_bbox:required", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].coverage_bbox", None), + expected_field="datasets[].coverage_bbox", + expected_check="required", + ), + Scenario( + id="sources::datasets[].coverage_bbox_min_length:array_min_length", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].coverage_bbox", []), + expected_field="datasets[].coverage_bbox_min_length", + expected_check="array_min_length", + ), + Scenario( + id="sources::datasets[].coverage_bbox_max_length:array_max_length", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + } + ] + }, + mutate=set_at_path("datasets[].coverage_bbox", [{}, {}, {}, {}, {}]), + expected_field="datasets[].coverage_bbox_max_length", + expected_check="array_max_length", + ), + Scenario( + id="sources::datasets[].url:url_format", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "url": "https://example.com/", + } + ] + }, + mutate=set_at_path("datasets[].url", "not-a-url"), + expected_field="datasets[].url", + expected_check="url_format", + ), + Scenario( + id="sources::datasets[].url:url_length", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "url": "https://example.com/", + } + ] + }, + mutate=set_at_path( + "datasets[].url", + "https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + ), + expected_field="datasets[].url", + expected_check="url_length", + ), + Scenario( + id="sources::datasets[].url_archived:url_format", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "url_archived": "https://example.com/", + } + ] + }, + mutate=set_at_path("datasets[].url_archived", "not-a-url"), + expected_field="datasets[].url_archived", + expected_check="url_format", + ), + Scenario( + id="sources::datasets[].url_archived:url_length", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "url_archived": "https://example.com/", + } + ] + }, + mutate=set_at_path( + "datasets[].url_archived", + "https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + ), + expected_field="datasets[].url_archived", + expected_check="url_length", + ), + Scenario( + id="sources::datasets[].data_download_url[]:url_format", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "data_download_url": ["https://example.com/"], + } + ] + }, + mutate=set_at_path("datasets[].data_download_url[]", "not-a-url"), + expected_field="datasets[].data_download_url[]", + expected_check="url_format", + valid_scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "data_download_url": [""], + } + ] + }, + ), + Scenario( + id="sources::datasets[].data_download_url[]:url_length", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "data_download_url": ["https://example.com/"], + } + ] + }, + mutate=set_at_path( + "datasets[].data_download_url[]", + "https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + ), + expected_field="datasets[].data_download_url[]", + expected_check="url_length", + valid_scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "data_download_url": [""], + } + ] + }, + ), + Scenario( + id="sources::datasets[].countries[]:country_code_alpha2", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "countries": ["US"], + } + ] + }, + mutate=set_at_path("datasets[].countries[]", "99"), + expected_field="datasets[].countries[]", + expected_check="country_code_alpha2", + valid_scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "countries": ["Global"], + } + ] + }, + ), + Scenario( + id="sources::datasets[].build_source:enum", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "build_source": "OpenAddresses", + } + ] + }, + mutate=set_at_path("datasets[].build_source", "__INVALID__"), + expected_field="datasets[].build_source", + expected_check="enum", + ), + Scenario( + id="sources::datasets[].update_type:enum", + scaffold={ + "datasets": [ + { + "source_name": "", + "source_dataset_name": "", + "data_url": "https://example.com/", + "data_url_archived": "https://example.com/", + "license_url": "https://example.com/", + "license_url_archived": "https://example.com/", + "license_type": "", + "license_text": "", + "license_attribution": "", + "coverage_bbox": [0.0, 0.0, 0.0, 0.0], + "update_type": "continuous", + } + ] + }, + mutate=set_at_path("datasets[].update_type", "__INVALID__"), + expected_field="datasets[].update_type", + expected_check="enum", + ), + Scenario( + id="sources::license_priority:required", + scaffold={}, + mutate=set_at_path("license_priority", None), + expected_field="license_priority", + expected_check="required", + ), + Scenario( + id="sources::license_priority{key}:pattern", + scaffold={}, + mutate=lambda row: mutate_map_key(row, "license_priority", "bad license!"), + expected_field="license_priority{key}", + expected_check="pattern", + ), + Scenario( + id="sources::license_priority{value}:bounds", + scaffold={}, + mutate=lambda row: mutate_map_value(row, "license_priority", -1), + expected_field="license_priority{value}", + expected_check="bounds", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return sources_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + SOURCES_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="sources", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + SOURCES_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="sources", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("sources::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("sources::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/base/__init__.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_bathymetry.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_bathymetry.py new file mode 100644 index 000000000..05ca0e028 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_bathymetry.py @@ -0,0 +1,463 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for bathymetry.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.base.bathymetry import ( + BATHYMETRY_SCHEMA, + bathymetry_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import mutate_unique_items +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "e1c02779-55d2-5d7e-8673-b7de1642ae68", + "geometry": "MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", + "theme": "base", + "type": "bathymetry", + "version": 0, + "depth": 0, +} + + +BASE_ROW_POPULATED: dict = { + "id": "e1c02779-55d2-5d7e-8673-b7de1642ae68", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", + "theme": "base", + "type": "bathymetry", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "depth": 0, + "cartography": {"prominence": 1, "min_zoom": 0, "max_zoom": 0, "sort_key": 0}, +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="bathymetry::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="bathymetry::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="bathymetry::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="bathymetry::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="bathymetry::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="bathymetry::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="bathymetry::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="bathymetry::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "POINT (0 0)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="bathymetry::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="bathymetry::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="bathymetry::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="bathymetry::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="bathymetry::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="bathymetry::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="bathymetry::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="bathymetry::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="bathymetry::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="bathymetry::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="bathymetry::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="bathymetry::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="bathymetry::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="bathymetry::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="bathymetry::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="bathymetry::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="bathymetry::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="bathymetry::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="bathymetry::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="bathymetry::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="bathymetry::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="bathymetry::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="bathymetry::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="bathymetry::depth:required", + scaffold={}, + mutate=set_at_path("depth", None), + expected_field="depth", + expected_check="required", + ), + Scenario( + id="bathymetry::depth:bounds", + scaffold={}, + mutate=set_at_path("depth", -1), + expected_field="depth", + expected_check="bounds", + ), + Scenario( + id="bathymetry::cartography.prominence_0:bounds", + scaffold={"cartography": {"prominence": 1}}, + mutate=set_at_path("cartography.prominence", 0), + expected_field="cartography.prominence_0", + expected_check="bounds", + ), + Scenario( + id="bathymetry::cartography.prominence_1:bounds", + scaffold={"cartography": {"prominence": 1}}, + mutate=set_at_path("cartography.prominence", 101), + expected_field="cartography.prominence_1", + expected_check="bounds", + ), + Scenario( + id="bathymetry::cartography.min_zoom_0:bounds", + scaffold={"cartography": {"min_zoom": 0}}, + mutate=set_at_path("cartography.min_zoom", -1), + expected_field="cartography.min_zoom_0", + expected_check="bounds", + ), + Scenario( + id="bathymetry::cartography.min_zoom_1:bounds", + scaffold={"cartography": {"min_zoom": 0}}, + mutate=set_at_path("cartography.min_zoom", 24), + expected_field="cartography.min_zoom_1", + expected_check="bounds", + ), + Scenario( + id="bathymetry::cartography.max_zoom_0:bounds", + scaffold={"cartography": {"max_zoom": 0}}, + mutate=set_at_path("cartography.max_zoom", -1), + expected_field="cartography.max_zoom_0", + expected_check="bounds", + ), + Scenario( + id="bathymetry::cartography.max_zoom_1:bounds", + scaffold={"cartography": {"max_zoom": 0}}, + mutate=set_at_path("cartography.max_zoom", 24), + expected_field="cartography.max_zoom_1", + expected_check="bounds", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return bathymetry_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + BATHYMETRY_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="bathymetry", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + BATHYMETRY_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="bathymetry", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("bathymetry::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("bathymetry::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_infrastructure.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_infrastructure.py new file mode 100644 index 000000000..ed8c656c8 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_infrastructure.py @@ -0,0 +1,730 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for infrastructure.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.base.infrastructure import ( + INFRASTRUCTURE_SCHEMA, + infrastructure_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_map_key, + mutate_map_value, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "e6cc8648-6bf9-5147-994f-621f86c9f103", + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "base", + "type": "infrastructure", + "version": 0, + "class": "aerialway_station", + "subtype": "aerialway", +} + + +BASE_ROW_POPULATED: dict = { + "id": "e6cc8648-6bf9-5147-994f-621f86c9f103", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "base", + "type": "infrastructure", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "class": "aerialway_station", + "subtype": "aerialway", + "height": 1.0, + "surface": "asphalt", + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, + "level": 0, + "source_tags": {}, + "wikidata": "Q42", +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="infrastructure::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="infrastructure::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="infrastructure::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="infrastructure::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="infrastructure::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="infrastructure::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="infrastructure::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="infrastructure::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "GEOMETRYCOLLECTION EMPTY"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="infrastructure::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="infrastructure::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="infrastructure::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="infrastructure::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="infrastructure::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="infrastructure::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="infrastructure::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="infrastructure::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="infrastructure::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="infrastructure::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="infrastructure::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="infrastructure::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="infrastructure::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="infrastructure::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="infrastructure::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="infrastructure::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="infrastructure::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="infrastructure::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="infrastructure::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="infrastructure::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="infrastructure::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="infrastructure::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="infrastructure::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="infrastructure::class:required", + scaffold={}, + mutate=set_at_path("class", None), + expected_field="class", + expected_check="required", + ), + Scenario( + id="infrastructure::class:enum", + scaffold={}, + mutate=set_at_path("class", "__INVALID__"), + expected_field="class", + expected_check="enum", + ), + Scenario( + id="infrastructure::subtype:required", + scaffold={}, + mutate=set_at_path("subtype", None), + expected_field="subtype", + expected_check="required", + ), + Scenario( + id="infrastructure::subtype:enum", + scaffold={}, + mutate=set_at_path("subtype", "__INVALID__"), + expected_field="subtype", + expected_check="enum", + ), + Scenario( + id="infrastructure::height:bounds", + scaffold={"height": 1.0}, + mutate=set_at_path("height", 0.0), + expected_field="height", + expected_check="bounds", + ), + Scenario( + id="infrastructure::surface:enum", + scaffold={"surface": "asphalt"}, + mutate=set_at_path("surface", "__INVALID__"), + expected_field="surface", + expected_check="enum", + ), + Scenario( + id="infrastructure::names.primary:required", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", None), + expected_field="names.primary", + expected_check="required", + ), + Scenario( + id="infrastructure::names.primary:string_min_length", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", ""), + expected_field="names.primary", + expected_check="string_min_length", + ), + Scenario( + id="infrastructure::names.primary:stripped", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", " has spaces "), + expected_field="names.primary", + expected_check="stripped", + ), + Scenario( + id="infrastructure::names.common{key}:language_tag", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_key(row, "names.common", "123"), + expected_field="names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="infrastructure::names.common{value}:stripped", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_value(row, "names.common", " has spaces "), + expected_field="names.common{value}", + expected_check="stripped", + ), + Scenario( + id="infrastructure::names.rules[].value:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", None), + expected_field="names.rules[].value", + expected_check="required", + ), + Scenario( + id="infrastructure::names.rules[].value:string_min_length", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", ""), + expected_field="names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="infrastructure::names.rules[].value:stripped", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", " has spaces "), + expected_field="names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="infrastructure::names.rules[].variant:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", None), + expected_field="names.rules[].variant", + expected_check="required", + ), + Scenario( + id="infrastructure::names.rules[].variant:enum", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", "__INVALID__"), + expected_field="names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="infrastructure::names.rules[].language:language_tag", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + }, + mutate=set_at_path("names.rules[].language", "123"), + expected_field="names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="infrastructure::names.rules[].perspectives.mode:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", None), + expected_field="names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="infrastructure::names.rules[].perspectives.mode:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", "__INVALID__"), + expected_field="names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="infrastructure::names.rules[].perspectives.countries:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", None), + expected_field="names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="infrastructure::names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", []), + expected_field="names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="infrastructure::names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=lambda row: mutate_unique_items( + row, "names.rules[].perspectives.countries" + ), + expected_field="names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="infrastructure::names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries[]", "99"), + expected_field="names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="infrastructure::names.rules[].between:linear_range_length", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.5]), + expected_field="names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="infrastructure::names.rules[].between:linear_range_bounds", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [1.5, 2.0]), + expected_field="names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="infrastructure::names.rules[].between:linear_range_order", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.8, 0.2]), + expected_field="names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="infrastructure::names.rules[].side:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + }, + mutate=set_at_path("names.rules[].side", "__INVALID__"), + expected_field="names.rules[].side", + expected_check="enum", + ), + Scenario( + id="infrastructure::wikidata:wikidata_id", + scaffold={"wikidata": "Q42"}, + mutate=set_at_path("wikidata", "P999"), + expected_field="wikidata", + expected_check="wikidata_id", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return infrastructure_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + INFRASTRUCTURE_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="infrastructure", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + INFRASTRUCTURE_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="infrastructure", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("infrastructure::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("infrastructure::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_land.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_land.py new file mode 100644 index 000000000..6e654c948 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_land.py @@ -0,0 +1,714 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for land.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.base.land import ( + LAND_SCHEMA, + land_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_map_key, + mutate_map_value, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "52a8b331-e001-5c79-8dab-dba632af0028", + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "base", + "type": "land", + "version": 0, +} + + +BASE_ROW_POPULATED: dict = { + "id": "52a8b331-e001-5c79-8dab-dba632af0028", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "base", + "type": "land", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "class": "archipelago", + "subtype": "crater", + "elevation": 9000, + "surface": "asphalt", + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, + "level": 0, + "source_tags": {}, + "wikidata": "Q42", +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="land::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="land::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="land::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="land::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="land::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="land::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="land::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="land::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "GEOMETRYCOLLECTION EMPTY"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="land::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="land::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="land::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="land::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="land::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="land::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="land::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="land::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="land::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="land::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="land::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="land::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="land::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="land::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="land::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="land::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="land::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="land::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="land::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="land::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="land::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="land::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="land::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="land::class:enum", + scaffold={"class": "archipelago"}, + mutate=set_at_path("class", "__INVALID__"), + expected_field="class", + expected_check="enum", + ), + Scenario( + id="land::subtype:enum", + scaffold={"subtype": "crater"}, + mutate=set_at_path("subtype", "__INVALID__"), + expected_field="subtype", + expected_check="enum", + ), + Scenario( + id="land::elevation:bounds", + scaffold={"elevation": 9000}, + mutate=set_at_path("elevation", 9001), + expected_field="elevation", + expected_check="bounds", + ), + Scenario( + id="land::surface:enum", + scaffold={"surface": "asphalt"}, + mutate=set_at_path("surface", "__INVALID__"), + expected_field="surface", + expected_check="enum", + ), + Scenario( + id="land::names.primary:required", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", None), + expected_field="names.primary", + expected_check="required", + ), + Scenario( + id="land::names.primary:string_min_length", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", ""), + expected_field="names.primary", + expected_check="string_min_length", + ), + Scenario( + id="land::names.primary:stripped", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", " has spaces "), + expected_field="names.primary", + expected_check="stripped", + ), + Scenario( + id="land::names.common{key}:language_tag", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_key(row, "names.common", "123"), + expected_field="names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="land::names.common{value}:stripped", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_value(row, "names.common", " has spaces "), + expected_field="names.common{value}", + expected_check="stripped", + ), + Scenario( + id="land::names.rules[].value:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", None), + expected_field="names.rules[].value", + expected_check="required", + ), + Scenario( + id="land::names.rules[].value:string_min_length", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", ""), + expected_field="names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="land::names.rules[].value:stripped", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", " has spaces "), + expected_field="names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="land::names.rules[].variant:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", None), + expected_field="names.rules[].variant", + expected_check="required", + ), + Scenario( + id="land::names.rules[].variant:enum", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", "__INVALID__"), + expected_field="names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="land::names.rules[].language:language_tag", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + }, + mutate=set_at_path("names.rules[].language", "123"), + expected_field="names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="land::names.rules[].perspectives.mode:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", None), + expected_field="names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="land::names.rules[].perspectives.mode:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", "__INVALID__"), + expected_field="names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="land::names.rules[].perspectives.countries:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", None), + expected_field="names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="land::names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", []), + expected_field="names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="land::names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=lambda row: mutate_unique_items( + row, "names.rules[].perspectives.countries" + ), + expected_field="names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="land::names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries[]", "99"), + expected_field="names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="land::names.rules[].between:linear_range_length", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.5]), + expected_field="names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="land::names.rules[].between:linear_range_bounds", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [1.5, 2.0]), + expected_field="names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="land::names.rules[].between:linear_range_order", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.8, 0.2]), + expected_field="names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="land::names.rules[].side:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + }, + mutate=set_at_path("names.rules[].side", "__INVALID__"), + expected_field="names.rules[].side", + expected_check="enum", + ), + Scenario( + id="land::wikidata:wikidata_id", + scaffold={"wikidata": "Q42"}, + mutate=set_at_path("wikidata", "P999"), + expected_field="wikidata", + expected_check="wikidata_id", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return land_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + LAND_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="land", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + LAND_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="land", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("land::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("land::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_land_cover.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_land_cover.py new file mode 100644 index 000000000..8a629e4ce --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_land_cover.py @@ -0,0 +1,463 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for land_cover.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.base.land_cover import ( + LAND_COVER_SCHEMA, + land_cover_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import mutate_unique_items +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "b03200e9-9f2f-52ac-bae8-e562b3fd26cc", + "geometry": "MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", + "theme": "base", + "type": "land_cover", + "version": 0, + "subtype": "barren", +} + + +BASE_ROW_POPULATED: dict = { + "id": "b03200e9-9f2f-52ac-bae8-e562b3fd26cc", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", + "theme": "base", + "type": "land_cover", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "subtype": "barren", + "cartography": {"prominence": 1, "min_zoom": 0, "max_zoom": 0, "sort_key": 0}, +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="land_cover::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="land_cover::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="land_cover::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="land_cover::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="land_cover::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="land_cover::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="land_cover::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="land_cover::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "POINT (0 0)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="land_cover::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="land_cover::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="land_cover::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="land_cover::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="land_cover::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="land_cover::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="land_cover::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="land_cover::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="land_cover::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="land_cover::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="land_cover::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="land_cover::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="land_cover::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="land_cover::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="land_cover::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="land_cover::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="land_cover::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="land_cover::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="land_cover::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="land_cover::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="land_cover::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="land_cover::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="land_cover::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="land_cover::subtype:required", + scaffold={}, + mutate=set_at_path("subtype", None), + expected_field="subtype", + expected_check="required", + ), + Scenario( + id="land_cover::subtype:enum", + scaffold={}, + mutate=set_at_path("subtype", "__INVALID__"), + expected_field="subtype", + expected_check="enum", + ), + Scenario( + id="land_cover::cartography.prominence_0:bounds", + scaffold={"cartography": {"prominence": 1}}, + mutate=set_at_path("cartography.prominence", 0), + expected_field="cartography.prominence_0", + expected_check="bounds", + ), + Scenario( + id="land_cover::cartography.prominence_1:bounds", + scaffold={"cartography": {"prominence": 1}}, + mutate=set_at_path("cartography.prominence", 101), + expected_field="cartography.prominence_1", + expected_check="bounds", + ), + Scenario( + id="land_cover::cartography.min_zoom_0:bounds", + scaffold={"cartography": {"min_zoom": 0}}, + mutate=set_at_path("cartography.min_zoom", -1), + expected_field="cartography.min_zoom_0", + expected_check="bounds", + ), + Scenario( + id="land_cover::cartography.min_zoom_1:bounds", + scaffold={"cartography": {"min_zoom": 0}}, + mutate=set_at_path("cartography.min_zoom", 24), + expected_field="cartography.min_zoom_1", + expected_check="bounds", + ), + Scenario( + id="land_cover::cartography.max_zoom_0:bounds", + scaffold={"cartography": {"max_zoom": 0}}, + mutate=set_at_path("cartography.max_zoom", -1), + expected_field="cartography.max_zoom_0", + expected_check="bounds", + ), + Scenario( + id="land_cover::cartography.max_zoom_1:bounds", + scaffold={"cartography": {"max_zoom": 0}}, + mutate=set_at_path("cartography.max_zoom", 24), + expected_field="cartography.max_zoom_1", + expected_check="bounds", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return land_cover_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + LAND_COVER_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="land_cover", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + LAND_COVER_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="land_cover", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("land_cover::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("land_cover::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_land_use.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_land_use.py new file mode 100644 index 000000000..d6469d3e8 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_land_use.py @@ -0,0 +1,730 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for land_use.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.base.land_use import ( + LAND_USE_SCHEMA, + land_use_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_map_key, + mutate_map_value, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "fe1e5b5f-3ae6-5c23-ba83-444a90ccd659", + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "base", + "type": "land_use", + "version": 0, + "class": "aboriginal_land", + "subtype": "agriculture", +} + + +BASE_ROW_POPULATED: dict = { + "id": "fe1e5b5f-3ae6-5c23-ba83-444a90ccd659", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "base", + "type": "land_use", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "class": "aboriginal_land", + "subtype": "agriculture", + "elevation": 9000, + "surface": "asphalt", + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, + "level": 0, + "source_tags": {}, + "wikidata": "Q42", +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="land_use::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="land_use::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="land_use::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="land_use::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="land_use::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="land_use::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="land_use::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="land_use::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "GEOMETRYCOLLECTION EMPTY"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="land_use::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="land_use::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="land_use::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="land_use::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="land_use::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="land_use::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="land_use::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="land_use::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="land_use::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="land_use::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="land_use::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="land_use::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="land_use::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="land_use::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="land_use::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="land_use::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="land_use::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="land_use::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="land_use::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="land_use::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="land_use::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="land_use::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="land_use::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="land_use::class:required", + scaffold={}, + mutate=set_at_path("class", None), + expected_field="class", + expected_check="required", + ), + Scenario( + id="land_use::class:enum", + scaffold={}, + mutate=set_at_path("class", "__INVALID__"), + expected_field="class", + expected_check="enum", + ), + Scenario( + id="land_use::subtype:required", + scaffold={}, + mutate=set_at_path("subtype", None), + expected_field="subtype", + expected_check="required", + ), + Scenario( + id="land_use::subtype:enum", + scaffold={}, + mutate=set_at_path("subtype", "__INVALID__"), + expected_field="subtype", + expected_check="enum", + ), + Scenario( + id="land_use::elevation:bounds", + scaffold={"elevation": 9000}, + mutate=set_at_path("elevation", 9001), + expected_field="elevation", + expected_check="bounds", + ), + Scenario( + id="land_use::surface:enum", + scaffold={"surface": "asphalt"}, + mutate=set_at_path("surface", "__INVALID__"), + expected_field="surface", + expected_check="enum", + ), + Scenario( + id="land_use::names.primary:required", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", None), + expected_field="names.primary", + expected_check="required", + ), + Scenario( + id="land_use::names.primary:string_min_length", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", ""), + expected_field="names.primary", + expected_check="string_min_length", + ), + Scenario( + id="land_use::names.primary:stripped", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", " has spaces "), + expected_field="names.primary", + expected_check="stripped", + ), + Scenario( + id="land_use::names.common{key}:language_tag", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_key(row, "names.common", "123"), + expected_field="names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="land_use::names.common{value}:stripped", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_value(row, "names.common", " has spaces "), + expected_field="names.common{value}", + expected_check="stripped", + ), + Scenario( + id="land_use::names.rules[].value:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", None), + expected_field="names.rules[].value", + expected_check="required", + ), + Scenario( + id="land_use::names.rules[].value:string_min_length", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", ""), + expected_field="names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="land_use::names.rules[].value:stripped", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", " has spaces "), + expected_field="names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="land_use::names.rules[].variant:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", None), + expected_field="names.rules[].variant", + expected_check="required", + ), + Scenario( + id="land_use::names.rules[].variant:enum", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", "__INVALID__"), + expected_field="names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="land_use::names.rules[].language:language_tag", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + }, + mutate=set_at_path("names.rules[].language", "123"), + expected_field="names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="land_use::names.rules[].perspectives.mode:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", None), + expected_field="names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="land_use::names.rules[].perspectives.mode:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", "__INVALID__"), + expected_field="names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="land_use::names.rules[].perspectives.countries:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", None), + expected_field="names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="land_use::names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", []), + expected_field="names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="land_use::names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=lambda row: mutate_unique_items( + row, "names.rules[].perspectives.countries" + ), + expected_field="names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="land_use::names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries[]", "99"), + expected_field="names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="land_use::names.rules[].between:linear_range_length", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.5]), + expected_field="names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="land_use::names.rules[].between:linear_range_bounds", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [1.5, 2.0]), + expected_field="names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="land_use::names.rules[].between:linear_range_order", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.8, 0.2]), + expected_field="names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="land_use::names.rules[].side:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + }, + mutate=set_at_path("names.rules[].side", "__INVALID__"), + expected_field="names.rules[].side", + expected_check="enum", + ), + Scenario( + id="land_use::wikidata:wikidata_id", + scaffold={"wikidata": "Q42"}, + mutate=set_at_path("wikidata", "P999"), + expected_field="wikidata", + expected_check="wikidata_id", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return land_use_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + LAND_USE_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="land_use", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + LAND_USE_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="land_use", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("land_use::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("land_use::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_water.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_water.py new file mode 100644 index 000000000..cf4c78434 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/base/test_water.py @@ -0,0 +1,700 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for water.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.base.water import ( + WATER_SCHEMA, + water_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_map_key, + mutate_map_value, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "a7a5e73a-79c0-55d7-ab4d-5f9fc65fe915", + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "base", + "type": "water", + "version": 0, +} + + +BASE_ROW_POPULATED: dict = { + "id": "a7a5e73a-79c0-55d7-ab4d-5f9fc65fe915", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "base", + "type": "water", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "class": "basin", + "subtype": "canal", + "is_intermittent": False, + "is_salt": False, + "level": 0, + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, + "source_tags": {}, + "wikidata": "Q42", +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="water::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="water::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="water::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="water::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="water::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="water::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="water::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="water::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "GEOMETRYCOLLECTION EMPTY"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="water::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="water::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="water::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="water::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="water::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="water::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="water::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="water::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="water::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="water::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="water::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="water::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="water::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="water::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="water::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="water::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="water::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="water::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="water::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="water::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="water::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="water::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="water::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="water::class:enum", + scaffold={"class": "basin"}, + mutate=set_at_path("class", "__INVALID__"), + expected_field="class", + expected_check="enum", + ), + Scenario( + id="water::subtype:enum", + scaffold={"subtype": "canal"}, + mutate=set_at_path("subtype", "__INVALID__"), + expected_field="subtype", + expected_check="enum", + ), + Scenario( + id="water::names.primary:required", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", None), + expected_field="names.primary", + expected_check="required", + ), + Scenario( + id="water::names.primary:string_min_length", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", ""), + expected_field="names.primary", + expected_check="string_min_length", + ), + Scenario( + id="water::names.primary:stripped", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", " has spaces "), + expected_field="names.primary", + expected_check="stripped", + ), + Scenario( + id="water::names.common{key}:language_tag", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_key(row, "names.common", "123"), + expected_field="names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="water::names.common{value}:stripped", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_value(row, "names.common", " has spaces "), + expected_field="names.common{value}", + expected_check="stripped", + ), + Scenario( + id="water::names.rules[].value:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", None), + expected_field="names.rules[].value", + expected_check="required", + ), + Scenario( + id="water::names.rules[].value:string_min_length", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", ""), + expected_field="names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="water::names.rules[].value:stripped", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", " has spaces "), + expected_field="names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="water::names.rules[].variant:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", None), + expected_field="names.rules[].variant", + expected_check="required", + ), + Scenario( + id="water::names.rules[].variant:enum", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", "__INVALID__"), + expected_field="names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="water::names.rules[].language:language_tag", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + }, + mutate=set_at_path("names.rules[].language", "123"), + expected_field="names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="water::names.rules[].perspectives.mode:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", None), + expected_field="names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="water::names.rules[].perspectives.mode:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", "__INVALID__"), + expected_field="names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="water::names.rules[].perspectives.countries:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", None), + expected_field="names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="water::names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", []), + expected_field="names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="water::names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=lambda row: mutate_unique_items( + row, "names.rules[].perspectives.countries" + ), + expected_field="names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="water::names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries[]", "99"), + expected_field="names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="water::names.rules[].between:linear_range_length", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.5]), + expected_field="names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="water::names.rules[].between:linear_range_bounds", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [1.5, 2.0]), + expected_field="names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="water::names.rules[].between:linear_range_order", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.8, 0.2]), + expected_field="names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="water::names.rules[].side:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + }, + mutate=set_at_path("names.rules[].side", "__INVALID__"), + expected_field="names.rules[].side", + expected_check="enum", + ), + Scenario( + id="water::wikidata:wikidata_id", + scaffold={"wikidata": "Q42"}, + mutate=set_at_path("wikidata", "P999"), + expected_field="wikidata", + expected_check="wikidata_id", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return water_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + WATER_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="water", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + WATER_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="water", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("water::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("water::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/buildings/__init__.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/buildings/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/buildings/test_building.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/buildings/test_building.py new file mode 100644 index 000000000..4587abd3d --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/buildings/test_building.py @@ -0,0 +1,788 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for building.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.buildings.building import ( + BUILDING_SCHEMA, + building_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_map_key, + mutate_map_value, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "f59ea25f-5910-56e0-b595-25dd9d65ef4b", + "geometry": "MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", + "theme": "buildings", + "type": "building", + "version": 0, +} + + +BASE_ROW_POPULATED: dict = { + "id": "f59ea25f-5910-56e0-b595-25dd9d65ef4b", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", + "theme": "buildings", + "type": "building", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "subtype": "agricultural", + "class": "agricultural", + "has_parts": False, + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, + "level": 0, + "height": 1.0, + "is_underground": False, + "num_floors": 1, + "num_floors_underground": 1, + "min_height": 0.0, + "min_floor": 1, + "facade_color": "#aabbcc", + "facade_material": "brick", + "roof_material": "concrete", + "roof_shape": "dome", + "roof_direction": 0.0, + "roof_orientation": "across", + "roof_color": "#aabbcc", + "roof_height": 0.0, +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="building::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="building::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="building::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="building::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="building::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="building::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="building::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="building::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "POINT (0 0)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="building::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="building::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="building::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="building::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="building::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="building::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="building::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="building::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="building::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="building::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="building::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="building::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="building::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="building::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="building::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="building::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="building::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="building::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="building::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="building::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="building::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="building::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="building::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="building::subtype:enum", + scaffold={"subtype": "agricultural"}, + mutate=set_at_path("subtype", "__INVALID__"), + expected_field="subtype", + expected_check="enum", + ), + Scenario( + id="building::class:enum", + scaffold={"class": "agricultural"}, + mutate=set_at_path("class", "__INVALID__"), + expected_field="class", + expected_check="enum", + ), + Scenario( + id="building::names.primary:required", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", None), + expected_field="names.primary", + expected_check="required", + ), + Scenario( + id="building::names.primary:string_min_length", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", ""), + expected_field="names.primary", + expected_check="string_min_length", + ), + Scenario( + id="building::names.primary:stripped", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", " has spaces "), + expected_field="names.primary", + expected_check="stripped", + ), + Scenario( + id="building::names.common{key}:language_tag", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_key(row, "names.common", "123"), + expected_field="names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="building::names.common{value}:stripped", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_value(row, "names.common", " has spaces "), + expected_field="names.common{value}", + expected_check="stripped", + ), + Scenario( + id="building::names.rules[].value:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", None), + expected_field="names.rules[].value", + expected_check="required", + ), + Scenario( + id="building::names.rules[].value:string_min_length", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", ""), + expected_field="names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="building::names.rules[].value:stripped", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", " has spaces "), + expected_field="names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="building::names.rules[].variant:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", None), + expected_field="names.rules[].variant", + expected_check="required", + ), + Scenario( + id="building::names.rules[].variant:enum", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", "__INVALID__"), + expected_field="names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="building::names.rules[].language:language_tag", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + }, + mutate=set_at_path("names.rules[].language", "123"), + expected_field="names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="building::names.rules[].perspectives.mode:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", None), + expected_field="names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="building::names.rules[].perspectives.mode:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", "__INVALID__"), + expected_field="names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="building::names.rules[].perspectives.countries:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", None), + expected_field="names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="building::names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", []), + expected_field="names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="building::names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=lambda row: mutate_unique_items( + row, "names.rules[].perspectives.countries" + ), + expected_field="names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="building::names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries[]", "99"), + expected_field="names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="building::names.rules[].between:linear_range_length", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.5]), + expected_field="names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="building::names.rules[].between:linear_range_bounds", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [1.5, 2.0]), + expected_field="names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="building::names.rules[].between:linear_range_order", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.8, 0.2]), + expected_field="names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="building::names.rules[].side:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + }, + mutate=set_at_path("names.rules[].side", "__INVALID__"), + expected_field="names.rules[].side", + expected_check="enum", + ), + Scenario( + id="building::height:bounds", + scaffold={"height": 1.0}, + mutate=set_at_path("height", 0.0), + expected_field="height", + expected_check="bounds", + ), + Scenario( + id="building::num_floors:bounds", + scaffold={"num_floors": 1}, + mutate=set_at_path("num_floors", 0), + expected_field="num_floors", + expected_check="bounds", + ), + Scenario( + id="building::num_floors_underground:bounds", + scaffold={"num_floors_underground": 1}, + mutate=set_at_path("num_floors_underground", 0), + expected_field="num_floors_underground", + expected_check="bounds", + ), + Scenario( + id="building::min_floor:bounds", + scaffold={"min_floor": 1}, + mutate=set_at_path("min_floor", 0), + expected_field="min_floor", + expected_check="bounds", + ), + Scenario( + id="building::facade_color:hex_color", + scaffold={"facade_color": "#aabbcc"}, + mutate=set_at_path("facade_color", "not-hex"), + expected_field="facade_color", + expected_check="hex_color", + ), + Scenario( + id="building::facade_material:enum", + scaffold={"facade_material": "brick"}, + mutate=set_at_path("facade_material", "__INVALID__"), + expected_field="facade_material", + expected_check="enum", + ), + Scenario( + id="building::roof_material:enum", + scaffold={"roof_material": "concrete"}, + mutate=set_at_path("roof_material", "__INVALID__"), + expected_field="roof_material", + expected_check="enum", + ), + Scenario( + id="building::roof_shape:enum", + scaffold={"roof_shape": "dome"}, + mutate=set_at_path("roof_shape", "__INVALID__"), + expected_field="roof_shape", + expected_check="enum", + ), + Scenario( + id="building::roof_direction_0:bounds", + scaffold={"roof_direction": 0.0}, + mutate=set_at_path("roof_direction", -1.0), + expected_field="roof_direction_0", + expected_check="bounds", + ), + Scenario( + id="building::roof_direction_1:bounds", + scaffold={"roof_direction": 0.0}, + mutate=set_at_path("roof_direction", 360.0), + expected_field="roof_direction_1", + expected_check="bounds", + ), + Scenario( + id="building::roof_orientation:enum", + scaffold={"roof_orientation": "across"}, + mutate=set_at_path("roof_orientation", "__INVALID__"), + expected_field="roof_orientation", + expected_check="enum", + ), + Scenario( + id="building::roof_color:hex_color", + scaffold={"roof_color": "#aabbcc"}, + mutate=set_at_path("roof_color", "not-hex"), + expected_field="roof_color", + expected_check="hex_color", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return building_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + BUILDING_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="building", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + BUILDING_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="building", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("building::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("building::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/buildings/test_building_part.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/buildings/test_building_part.py new file mode 100644 index 000000000..5cbf03b64 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/buildings/test_building_part.py @@ -0,0 +1,794 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for building_part.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.buildings.building_part import ( + BUILDING_PART_SCHEMA, + building_part_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_map_key, + mutate_map_value, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "c039cf20-2e1c-5116-a393-4d834e447d46", + "geometry": "MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", + "theme": "buildings", + "type": "building_part", + "version": 0, + "building_id": "a", +} + + +BASE_ROW_POPULATED: dict = { + "id": "c039cf20-2e1c-5116-a393-4d834e447d46", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", + "theme": "buildings", + "type": "building_part", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "building_id": "a", + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, + "level": 0, + "height": 1.0, + "is_underground": False, + "num_floors": 1, + "num_floors_underground": 1, + "min_height": 0.0, + "min_floor": 1, + "facade_color": "#aabbcc", + "facade_material": "brick", + "roof_material": "concrete", + "roof_shape": "dome", + "roof_direction": 0.0, + "roof_orientation": "across", + "roof_color": "#aabbcc", + "roof_height": 0.0, +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="building_part::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="building_part::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="building_part::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="building_part::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="building_part::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="building_part::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="building_part::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="building_part::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "POINT (0 0)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="building_part::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="building_part::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="building_part::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="building_part::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="building_part::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="building_part::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="building_part::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="building_part::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="building_part::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="building_part::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="building_part::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="building_part::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="building_part::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="building_part::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="building_part::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="building_part::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="building_part::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="building_part::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="building_part::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="building_part::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="building_part::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="building_part::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="building_part::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="building_part::building_id:required", + scaffold={}, + mutate=set_at_path("building_id", None), + expected_field="building_id", + expected_check="required", + ), + Scenario( + id="building_part::building_id:string_min_length", + scaffold={}, + mutate=set_at_path("building_id", ""), + expected_field="building_id", + expected_check="string_min_length", + ), + Scenario( + id="building_part::building_id:no_whitespace", + scaffold={}, + mutate=set_at_path("building_id", "has whitespace"), + expected_field="building_id", + expected_check="no_whitespace", + ), + Scenario( + id="building_part::names.primary:required", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", None), + expected_field="names.primary", + expected_check="required", + ), + Scenario( + id="building_part::names.primary:string_min_length", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", ""), + expected_field="names.primary", + expected_check="string_min_length", + ), + Scenario( + id="building_part::names.primary:stripped", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", " has spaces "), + expected_field="names.primary", + expected_check="stripped", + ), + Scenario( + id="building_part::names.common{key}:language_tag", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_key(row, "names.common", "123"), + expected_field="names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="building_part::names.common{value}:stripped", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_value(row, "names.common", " has spaces "), + expected_field="names.common{value}", + expected_check="stripped", + ), + Scenario( + id="building_part::names.rules[].value:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", None), + expected_field="names.rules[].value", + expected_check="required", + ), + Scenario( + id="building_part::names.rules[].value:string_min_length", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", ""), + expected_field="names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="building_part::names.rules[].value:stripped", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", " has spaces "), + expected_field="names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="building_part::names.rules[].variant:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", None), + expected_field="names.rules[].variant", + expected_check="required", + ), + Scenario( + id="building_part::names.rules[].variant:enum", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", "__INVALID__"), + expected_field="names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="building_part::names.rules[].language:language_tag", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + }, + mutate=set_at_path("names.rules[].language", "123"), + expected_field="names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="building_part::names.rules[].perspectives.mode:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", None), + expected_field="names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="building_part::names.rules[].perspectives.mode:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", "__INVALID__"), + expected_field="names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="building_part::names.rules[].perspectives.countries:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", None), + expected_field="names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="building_part::names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", []), + expected_field="names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="building_part::names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=lambda row: mutate_unique_items( + row, "names.rules[].perspectives.countries" + ), + expected_field="names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="building_part::names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries[]", "99"), + expected_field="names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="building_part::names.rules[].between:linear_range_length", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.5]), + expected_field="names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="building_part::names.rules[].between:linear_range_bounds", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [1.5, 2.0]), + expected_field="names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="building_part::names.rules[].between:linear_range_order", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.8, 0.2]), + expected_field="names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="building_part::names.rules[].side:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + }, + mutate=set_at_path("names.rules[].side", "__INVALID__"), + expected_field="names.rules[].side", + expected_check="enum", + ), + Scenario( + id="building_part::height:bounds", + scaffold={"height": 1.0}, + mutate=set_at_path("height", 0.0), + expected_field="height", + expected_check="bounds", + ), + Scenario( + id="building_part::num_floors:bounds", + scaffold={"num_floors": 1}, + mutate=set_at_path("num_floors", 0), + expected_field="num_floors", + expected_check="bounds", + ), + Scenario( + id="building_part::num_floors_underground:bounds", + scaffold={"num_floors_underground": 1}, + mutate=set_at_path("num_floors_underground", 0), + expected_field="num_floors_underground", + expected_check="bounds", + ), + Scenario( + id="building_part::min_floor:bounds", + scaffold={"min_floor": 1}, + mutate=set_at_path("min_floor", 0), + expected_field="min_floor", + expected_check="bounds", + ), + Scenario( + id="building_part::facade_color:hex_color", + scaffold={"facade_color": "#aabbcc"}, + mutate=set_at_path("facade_color", "not-hex"), + expected_field="facade_color", + expected_check="hex_color", + ), + Scenario( + id="building_part::facade_material:enum", + scaffold={"facade_material": "brick"}, + mutate=set_at_path("facade_material", "__INVALID__"), + expected_field="facade_material", + expected_check="enum", + ), + Scenario( + id="building_part::roof_material:enum", + scaffold={"roof_material": "concrete"}, + mutate=set_at_path("roof_material", "__INVALID__"), + expected_field="roof_material", + expected_check="enum", + ), + Scenario( + id="building_part::roof_shape:enum", + scaffold={"roof_shape": "dome"}, + mutate=set_at_path("roof_shape", "__INVALID__"), + expected_field="roof_shape", + expected_check="enum", + ), + Scenario( + id="building_part::roof_direction_0:bounds", + scaffold={"roof_direction": 0.0}, + mutate=set_at_path("roof_direction", -1.0), + expected_field="roof_direction_0", + expected_check="bounds", + ), + Scenario( + id="building_part::roof_direction_1:bounds", + scaffold={"roof_direction": 0.0}, + mutate=set_at_path("roof_direction", 360.0), + expected_field="roof_direction_1", + expected_check="bounds", + ), + Scenario( + id="building_part::roof_orientation:enum", + scaffold={"roof_orientation": "across"}, + mutate=set_at_path("roof_orientation", "__INVALID__"), + expected_field="roof_orientation", + expected_check="enum", + ), + Scenario( + id="building_part::roof_color:hex_color", + scaffold={"roof_color": "#aabbcc"}, + mutate=set_at_path("roof_color", "not-hex"), + expected_field="roof_color", + expected_check="hex_color", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return building_part_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + BUILDING_PART_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="building_part", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + BUILDING_PART_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="building_part", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("building_part::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("building_part::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/divisions/__init__.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/divisions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/divisions/test_division.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/divisions/test_division.py new file mode 100644 index 000000000..42616745e --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/divisions/test_division.py @@ -0,0 +1,1141 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for division.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.divisions.division import ( + DIVISION_SCHEMA, + division_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_forbid_if, + mutate_map_key, + mutate_map_value, + mutate_require_if, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "names": {"primary": "a"}, + "id": "97a2a97d-1eb8-5161-9ae5-bfb82594ed67", + "geometry": "POINT (0 0)", + "theme": "divisions", + "type": "division", + "version": 0, + "subtype": "country", + "country": "US", + "hierarchies": [[{"division_id": "a", "subtype": "country", "name": "a"}]], + "admin_level": 0, +} + + +BASE_ROW_POPULATED: dict = { + "cartography": {"prominence": 1, "min_zoom": 0, "max_zoom": 0, "sort_key": 0}, + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, + "id": "97a2a97d-1eb8-5161-9ae5-bfb82594ed67", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "POINT (0 0)", + "theme": "divisions", + "type": "division", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "subtype": "country", + "country": "US", + "hierarchies": [[{"division_id": "a", "subtype": "country", "name": "a"}]], + "admin_level": 0, + "class": "megacity", + "local_type": {"en": "clean"}, + "region": "US-CA", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "norms": {"driving_side": "left"}, + "population": 0, + "capital_division_ids": ["a"], + "capital_of_divisions": [{"division_id": "a", "subtype": "country"}], + "wikidata": "Q42", +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="division::cartography.prominence_0:bounds", + scaffold={"cartography": {"prominence": 1}}, + mutate=set_at_path("cartography.prominence", 0), + expected_field="cartography.prominence_0", + expected_check="bounds", + ), + Scenario( + id="division::cartography.prominence_1:bounds", + scaffold={"cartography": {"prominence": 1}}, + mutate=set_at_path("cartography.prominence", 101), + expected_field="cartography.prominence_1", + expected_check="bounds", + ), + Scenario( + id="division::cartography.min_zoom_0:bounds", + scaffold={"cartography": {"min_zoom": 0}}, + mutate=set_at_path("cartography.min_zoom", -1), + expected_field="cartography.min_zoom_0", + expected_check="bounds", + ), + Scenario( + id="division::cartography.min_zoom_1:bounds", + scaffold={"cartography": {"min_zoom": 0}}, + mutate=set_at_path("cartography.min_zoom", 24), + expected_field="cartography.min_zoom_1", + expected_check="bounds", + ), + Scenario( + id="division::cartography.max_zoom_0:bounds", + scaffold={"cartography": {"max_zoom": 0}}, + mutate=set_at_path("cartography.max_zoom", -1), + expected_field="cartography.max_zoom_0", + expected_check="bounds", + ), + Scenario( + id="division::cartography.max_zoom_1:bounds", + scaffold={"cartography": {"max_zoom": 0}}, + mutate=set_at_path("cartography.max_zoom", 24), + expected_field="cartography.max_zoom_1", + expected_check="bounds", + ), + Scenario( + id="division::names:required", + scaffold={}, + mutate=set_at_path("names", None), + expected_field="names", + expected_check="required", + ), + Scenario( + id="division::names.primary:required", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", None), + expected_field="names.primary", + expected_check="required", + ), + Scenario( + id="division::names.primary:string_min_length", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", ""), + expected_field="names.primary", + expected_check="string_min_length", + ), + Scenario( + id="division::names.primary:stripped", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", " has spaces "), + expected_field="names.primary", + expected_check="stripped", + ), + Scenario( + id="division::names.common{key}:language_tag", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_key(row, "names.common", "123"), + expected_field="names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="division::names.common{value}:stripped", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_value(row, "names.common", " has spaces "), + expected_field="names.common{value}", + expected_check="stripped", + ), + Scenario( + id="division::names.rules[].value:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", None), + expected_field="names.rules[].value", + expected_check="required", + ), + Scenario( + id="division::names.rules[].value:string_min_length", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", ""), + expected_field="names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="division::names.rules[].value:stripped", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", " has spaces "), + expected_field="names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="division::names.rules[].variant:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", None), + expected_field="names.rules[].variant", + expected_check="required", + ), + Scenario( + id="division::names.rules[].variant:enum", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", "__INVALID__"), + expected_field="names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="division::names.rules[].language:language_tag", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + }, + mutate=set_at_path("names.rules[].language", "123"), + expected_field="names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="division::names.rules[].perspectives.mode:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", None), + expected_field="names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="division::names.rules[].perspectives.mode:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", "__INVALID__"), + expected_field="names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="division::names.rules[].perspectives.countries:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", None), + expected_field="names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="division::names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", []), + expected_field="names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="division::names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=lambda row: mutate_unique_items( + row, "names.rules[].perspectives.countries" + ), + expected_field="names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="division::names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries[]", "99"), + expected_field="names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="division::names.rules[].between:linear_range_length", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.5]), + expected_field="names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="division::names.rules[].between:linear_range_bounds", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [1.5, 2.0]), + expected_field="names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="division::names.rules[].between:linear_range_order", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.8, 0.2]), + expected_field="names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="division::names.rules[].side:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + }, + mutate=set_at_path("names.rules[].side", "__INVALID__"), + expected_field="names.rules[].side", + expected_check="enum", + ), + Scenario( + id="division::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="division::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="division::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="division::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="division::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="division::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="division::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="division::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "LINESTRING (0 0, 1 1)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="division::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="division::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="division::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="division::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="division::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="division::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="division::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="division::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="division::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="division::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="division::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="division::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="division::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="division::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="division::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="division::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="division::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="division::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="division::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="division::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="division::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="division::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="division::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="division::subtype:required", + scaffold={}, + mutate=set_at_path("subtype", None), + expected_field="subtype", + expected_check="required", + ), + Scenario( + id="division::subtype:enum", + scaffold={}, + mutate=set_at_path("subtype", "__INVALID__"), + expected_field="subtype", + expected_check="enum", + ), + Scenario( + id="division::country:required", + scaffold={}, + mutate=set_at_path("country", None), + expected_field="country", + expected_check="required", + ), + Scenario( + id="division::country:country_code_alpha2", + scaffold={}, + mutate=set_at_path("country", "99"), + expected_field="country", + expected_check="country_code_alpha2", + ), + Scenario( + id="division::hierarchies:required", + scaffold={}, + mutate=set_at_path("hierarchies", None), + expected_field="hierarchies", + expected_check="required", + ), + Scenario( + id="division::hierarchies_min_length:array_min_length", + scaffold={}, + mutate=set_at_path("hierarchies", []), + expected_field="hierarchies_min_length", + expected_check="array_min_length", + ), + Scenario( + id="division::hierarchies_unique:struct_unique", + scaffold={}, + mutate=lambda row: mutate_unique_items(row, "hierarchies"), + expected_field="hierarchies_unique", + expected_check="struct_unique", + ), + Scenario( + id="division::hierarchies[]_min_length:array_min_length", + scaffold={}, + mutate=set_at_path("hierarchies[]", []), + expected_field="hierarchies[]_min_length", + expected_check="array_min_length", + ), + Scenario( + id="division::hierarchies[]_unique:struct_unique", + scaffold={}, + mutate=lambda row: mutate_unique_items(row, "hierarchies[]"), + expected_field="hierarchies[]_unique", + expected_check="struct_unique", + ), + Scenario( + id="division::hierarchies[][].division_id:required", + scaffold={ + "hierarchies": [[{"division_id": "a", "subtype": "country", "name": "a"}]] + }, + mutate=set_at_path("hierarchies[][].division_id", None), + expected_field="hierarchies[][].division_id", + expected_check="required", + ), + Scenario( + id="division::hierarchies[][].division_id:string_min_length", + scaffold={ + "hierarchies": [[{"division_id": "a", "subtype": "country", "name": "a"}]] + }, + mutate=set_at_path("hierarchies[][].division_id", ""), + expected_field="hierarchies[][].division_id", + expected_check="string_min_length", + ), + Scenario( + id="division::hierarchies[][].division_id:no_whitespace", + scaffold={ + "hierarchies": [[{"division_id": "a", "subtype": "country", "name": "a"}]] + }, + mutate=set_at_path("hierarchies[][].division_id", "has whitespace"), + expected_field="hierarchies[][].division_id", + expected_check="no_whitespace", + ), + Scenario( + id="division::hierarchies[][].subtype:required", + scaffold={ + "hierarchies": [[{"division_id": "a", "subtype": "country", "name": "a"}]] + }, + mutate=set_at_path("hierarchies[][].subtype", None), + expected_field="hierarchies[][].subtype", + expected_check="required", + ), + Scenario( + id="division::hierarchies[][].subtype:enum", + scaffold={ + "hierarchies": [[{"division_id": "a", "subtype": "country", "name": "a"}]] + }, + mutate=set_at_path("hierarchies[][].subtype", "__INVALID__"), + expected_field="hierarchies[][].subtype", + expected_check="enum", + ), + Scenario( + id="division::hierarchies[][].name:required", + scaffold={ + "hierarchies": [[{"division_id": "a", "subtype": "country", "name": "a"}]] + }, + mutate=set_at_path("hierarchies[][].name", None), + expected_field="hierarchies[][].name", + expected_check="required", + ), + Scenario( + id="division::hierarchies[][].name:string_min_length", + scaffold={ + "hierarchies": [[{"division_id": "a", "subtype": "country", "name": "a"}]] + }, + mutate=set_at_path("hierarchies[][].name", ""), + expected_field="hierarchies[][].name", + expected_check="string_min_length", + ), + Scenario( + id="division::hierarchies[][].name:stripped", + scaffold={ + "hierarchies": [[{"division_id": "a", "subtype": "country", "name": "a"}]] + }, + mutate=set_at_path("hierarchies[][].name", " has spaces "), + expected_field="hierarchies[][].name", + expected_check="stripped", + ), + Scenario( + id="division::parent_division_id:string_min_length", + scaffold={"subtype": "dependency", "parent_division_id": "a"}, + mutate=set_at_path("parent_division_id", ""), + expected_field="parent_division_id", + expected_check="string_min_length", + ), + Scenario( + id="division::parent_division_id:no_whitespace", + scaffold={"subtype": "dependency", "parent_division_id": "a"}, + mutate=set_at_path("parent_division_id", "has whitespace"), + expected_field="parent_division_id", + expected_check="no_whitespace", + ), + Scenario( + id="division::admin_level_0:bounds", + scaffold={"admin_level": 0}, + mutate=set_at_path("admin_level", -1), + expected_field="admin_level_0", + expected_check="bounds", + ), + Scenario( + id="division::admin_level_1:bounds", + scaffold={"admin_level": 0}, + mutate=set_at_path("admin_level", 17), + expected_field="admin_level_1", + expected_check="bounds", + ), + Scenario( + id="division::class:enum", + scaffold={"class": "megacity"}, + mutate=set_at_path("class", "__INVALID__"), + expected_field="class", + expected_check="enum", + ), + Scenario( + id="division::local_type{key}:language_tag", + scaffold={"local_type": {"en": "clean"}}, + mutate=lambda row: mutate_map_key(row, "local_type", "123"), + expected_field="local_type{key}", + expected_check="language_tag", + ), + Scenario( + id="division::local_type{value}:stripped", + scaffold={"local_type": {"en": "clean"}}, + mutate=lambda row: mutate_map_value(row, "local_type", " has spaces "), + expected_field="local_type{value}", + expected_check="stripped", + ), + Scenario( + id="division::region:region_code", + scaffold={"region": "US-CA"}, + mutate=set_at_path("region", "99-999"), + expected_field="region", + expected_check="region_code", + ), + Scenario( + id="division::perspectives.mode:required", + scaffold={"perspectives": {"mode": "accepted_by", "countries": ["US"]}}, + mutate=set_at_path("perspectives.mode", None), + expected_field="perspectives.mode", + expected_check="required", + ), + Scenario( + id="division::perspectives.mode:enum", + scaffold={"perspectives": {"mode": "accepted_by", "countries": ["US"]}}, + mutate=set_at_path("perspectives.mode", "__INVALID__"), + expected_field="perspectives.mode", + expected_check="enum", + ), + Scenario( + id="division::perspectives.countries:required", + scaffold={"perspectives": {"mode": "accepted_by", "countries": ["US"]}}, + mutate=set_at_path("perspectives.countries", None), + expected_field="perspectives.countries", + expected_check="required", + ), + Scenario( + id="division::perspectives.countries_min_length:array_min_length", + scaffold={"perspectives": {"mode": "accepted_by", "countries": ["US"]}}, + mutate=set_at_path("perspectives.countries", []), + expected_field="perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="division::perspectives.countries_unique:struct_unique", + scaffold={"perspectives": {"mode": "accepted_by", "countries": ["US"]}}, + mutate=lambda row: mutate_unique_items(row, "perspectives.countries"), + expected_field="perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="division::perspectives.countries[]:country_code_alpha2", + scaffold={"perspectives": {"mode": "accepted_by", "countries": ["US"]}}, + mutate=set_at_path("perspectives.countries[]", "99"), + expected_field="perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="division::norms.driving_side:enum", + scaffold={"norms": {"driving_side": "left"}}, + mutate=set_at_path("norms.driving_side", "__INVALID__"), + expected_field="norms.driving_side", + expected_check="enum", + ), + Scenario( + id="division::population:bounds", + scaffold={"population": 0}, + mutate=set_at_path("population", -1), + expected_field="population", + expected_check="bounds", + ), + Scenario( + id="division::capital_division_ids_min_length:array_min_length", + scaffold={"capital_division_ids": ["a"]}, + mutate=set_at_path("capital_division_ids", []), + expected_field="capital_division_ids_min_length", + expected_check="array_min_length", + ), + Scenario( + id="division::capital_division_ids_unique:struct_unique", + scaffold={"capital_division_ids": ["a"]}, + mutate=lambda row: mutate_unique_items(row, "capital_division_ids"), + expected_field="capital_division_ids_unique", + expected_check="struct_unique", + ), + Scenario( + id="division::capital_division_ids[]:string_min_length", + scaffold={"capital_division_ids": ["a"]}, + mutate=set_at_path("capital_division_ids[]", ""), + expected_field="capital_division_ids[]", + expected_check="string_min_length", + ), + Scenario( + id="division::capital_division_ids[]:no_whitespace", + scaffold={"capital_division_ids": ["a"]}, + mutate=set_at_path("capital_division_ids[]", "has whitespace"), + expected_field="capital_division_ids[]", + expected_check="no_whitespace", + ), + Scenario( + id="division::capital_of_divisions_min_length:array_min_length", + scaffold={"capital_of_divisions": [{"division_id": "a", "subtype": "country"}]}, + mutate=set_at_path("capital_of_divisions", []), + expected_field="capital_of_divisions_min_length", + expected_check="array_min_length", + ), + Scenario( + id="division::capital_of_divisions_unique:struct_unique", + scaffold={"capital_of_divisions": [{"division_id": "a", "subtype": "country"}]}, + mutate=lambda row: mutate_unique_items(row, "capital_of_divisions"), + expected_field="capital_of_divisions_unique", + expected_check="struct_unique", + ), + Scenario( + id="division::capital_of_divisions[].division_id:required", + scaffold={"capital_of_divisions": [{"division_id": "a", "subtype": "country"}]}, + mutate=set_at_path("capital_of_divisions[].division_id", None), + expected_field="capital_of_divisions[].division_id", + expected_check="required", + ), + Scenario( + id="division::capital_of_divisions[].division_id:string_min_length", + scaffold={"capital_of_divisions": [{"division_id": "a", "subtype": "country"}]}, + mutate=set_at_path("capital_of_divisions[].division_id", ""), + expected_field="capital_of_divisions[].division_id", + expected_check="string_min_length", + ), + Scenario( + id="division::capital_of_divisions[].division_id:no_whitespace", + scaffold={"capital_of_divisions": [{"division_id": "a", "subtype": "country"}]}, + mutate=set_at_path("capital_of_divisions[].division_id", "has whitespace"), + expected_field="capital_of_divisions[].division_id", + expected_check="no_whitespace", + ), + Scenario( + id="division::capital_of_divisions[].subtype:required", + scaffold={"capital_of_divisions": [{"division_id": "a", "subtype": "country"}]}, + mutate=set_at_path("capital_of_divisions[].subtype", None), + expected_field="capital_of_divisions[].subtype", + expected_check="required", + ), + Scenario( + id="division::capital_of_divisions[].subtype:enum", + scaffold={"capital_of_divisions": [{"division_id": "a", "subtype": "country"}]}, + mutate=set_at_path("capital_of_divisions[].subtype", "__INVALID__"), + expected_field="capital_of_divisions[].subtype", + expected_check="enum", + ), + Scenario( + id="division::wikidata:wikidata_id", + scaffold={"wikidata": "Q42"}, + mutate=set_at_path("wikidata", "P999"), + expected_field="wikidata", + expected_check="wikidata_id", + ), + Scenario( + id="division::model:require_if:0", + scaffold={}, + mutate=lambda row: mutate_require_if(row, ["admin_level"], "subtype", "county"), + expected_field="admin_level_required_0", + expected_check="require_if", + ), + Scenario( + id="division::model:require_if:1", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["admin_level"], "subtype", "macrocounty" + ), + expected_field="admin_level_required_1", + expected_check="require_if", + ), + Scenario( + id="division::model:require_if:2", + scaffold={}, + mutate=lambda row: mutate_require_if(row, ["admin_level"], "subtype", "region"), + expected_field="admin_level_required_2", + expected_check="require_if", + ), + Scenario( + id="division::model:require_if:3", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["admin_level"], "subtype", "macroregion" + ), + expected_field="admin_level_required_3", + expected_check="require_if", + ), + Scenario( + id="division::model:require_if:4", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["admin_level"], "subtype", "dependency" + ), + expected_field="admin_level_required_4", + expected_check="require_if", + ), + Scenario( + id="division::model:require_if:5", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["admin_level"], "subtype", "country" + ), + expected_field="admin_level_required_5", + expected_check="require_if", + ), + Scenario( + id="division::model:require_if:6", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["parent_division_id"], "subtype", "country", negate=True + ), + expected_field="parent_division_id_required", + expected_check="require_if", + ), + Scenario( + id="division::model:forbid_if:7", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, ["parent_division_id"], "subtype", "country" + ), + expected_field="parent_division_id_forbidden", + expected_check="forbid_if", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return division_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + DIVISION_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="division", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + DIVISION_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="division", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("division::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("division::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/divisions/test_division_area.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/divisions/test_division_area.py new file mode 100644 index 000000000..0c0e1517a --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/divisions/test_division_area.py @@ -0,0 +1,839 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for division_area.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.divisions.division_area import ( + DIVISION_AREA_SCHEMA, + division_area_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_map_key, + mutate_map_value, + mutate_require_any_true, + mutate_require_if, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "names": {"primary": "a"}, + "id": "4619f66f-2d01-5776-ba67-01e9f3ccd9d7", + "geometry": "MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", + "theme": "divisions", + "type": "division_area", + "version": 0, + "subtype": "country", + "class": "land", + "division_id": "a", + "country": "US", + "is_land": True, + "admin_level": 0, +} + + +BASE_ROW_POPULATED: dict = { + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, + "id": "4619f66f-2d01-5776-ba67-01e9f3ccd9d7", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "MULTIPOLYGON (((0 0, 1 0, 1 1, 0 1, 0 0)))", + "theme": "divisions", + "type": "division_area", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "subtype": "country", + "class": "land", + "is_land": True, + "is_territorial": False, + "division_id": "a", + "country": "US", + "region": "US-CA", + "admin_level": 0, +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="division_area::names:required", + scaffold={}, + mutate=set_at_path("names", None), + expected_field="names", + expected_check="required", + ), + Scenario( + id="division_area::names.primary:required", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", None), + expected_field="names.primary", + expected_check="required", + ), + Scenario( + id="division_area::names.primary:string_min_length", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", ""), + expected_field="names.primary", + expected_check="string_min_length", + ), + Scenario( + id="division_area::names.primary:stripped", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", " has spaces "), + expected_field="names.primary", + expected_check="stripped", + ), + Scenario( + id="division_area::names.common{key}:language_tag", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_key(row, "names.common", "123"), + expected_field="names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="division_area::names.common{value}:stripped", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_value(row, "names.common", " has spaces "), + expected_field="names.common{value}", + expected_check="stripped", + ), + Scenario( + id="division_area::names.rules[].value:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", None), + expected_field="names.rules[].value", + expected_check="required", + ), + Scenario( + id="division_area::names.rules[].value:string_min_length", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", ""), + expected_field="names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="division_area::names.rules[].value:stripped", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", " has spaces "), + expected_field="names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="division_area::names.rules[].variant:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", None), + expected_field="names.rules[].variant", + expected_check="required", + ), + Scenario( + id="division_area::names.rules[].variant:enum", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", "__INVALID__"), + expected_field="names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="division_area::names.rules[].language:language_tag", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + }, + mutate=set_at_path("names.rules[].language", "123"), + expected_field="names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="division_area::names.rules[].perspectives.mode:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", None), + expected_field="names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="division_area::names.rules[].perspectives.mode:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", "__INVALID__"), + expected_field="names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="division_area::names.rules[].perspectives.countries:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", None), + expected_field="names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="division_area::names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", []), + expected_field="names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="division_area::names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=lambda row: mutate_unique_items( + row, "names.rules[].perspectives.countries" + ), + expected_field="names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="division_area::names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries[]", "99"), + expected_field="names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="division_area::names.rules[].between:linear_range_length", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.5]), + expected_field="names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="division_area::names.rules[].between:linear_range_bounds", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [1.5, 2.0]), + expected_field="names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="division_area::names.rules[].between:linear_range_order", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.8, 0.2]), + expected_field="names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="division_area::names.rules[].side:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + }, + mutate=set_at_path("names.rules[].side", "__INVALID__"), + expected_field="names.rules[].side", + expected_check="enum", + ), + Scenario( + id="division_area::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="division_area::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="division_area::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="division_area::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="division_area::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="division_area::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="division_area::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="division_area::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "POINT (0 0)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="division_area::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="division_area::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="division_area::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="division_area::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="division_area::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="division_area::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="division_area::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="division_area::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="division_area::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="division_area::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="division_area::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="division_area::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="division_area::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="division_area::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="division_area::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="division_area::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="division_area::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="division_area::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="division_area::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="division_area::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="division_area::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="division_area::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="division_area::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="division_area::subtype:required", + scaffold={}, + mutate=set_at_path("subtype", None), + expected_field="subtype", + expected_check="required", + ), + Scenario( + id="division_area::subtype:enum", + scaffold={}, + mutate=set_at_path("subtype", "__INVALID__"), + expected_field="subtype", + expected_check="enum", + ), + Scenario( + id="division_area::class:required", + scaffold={}, + mutate=set_at_path("class", None), + expected_field="class", + expected_check="required", + ), + Scenario( + id="division_area::class:enum", + scaffold={}, + mutate=set_at_path("class", "__INVALID__"), + expected_field="class", + expected_check="enum", + ), + Scenario( + id="division_area::division_id:required", + scaffold={}, + mutate=set_at_path("division_id", None), + expected_field="division_id", + expected_check="required", + ), + Scenario( + id="division_area::division_id:string_min_length", + scaffold={}, + mutate=set_at_path("division_id", ""), + expected_field="division_id", + expected_check="string_min_length", + ), + Scenario( + id="division_area::division_id:no_whitespace", + scaffold={}, + mutate=set_at_path("division_id", "has whitespace"), + expected_field="division_id", + expected_check="no_whitespace", + ), + Scenario( + id="division_area::country:required", + scaffold={}, + mutate=set_at_path("country", None), + expected_field="country", + expected_check="required", + ), + Scenario( + id="division_area::country:country_code_alpha2", + scaffold={}, + mutate=set_at_path("country", "99"), + expected_field="country", + expected_check="country_code_alpha2", + ), + Scenario( + id="division_area::region:region_code", + scaffold={"region": "US-CA"}, + mutate=set_at_path("region", "99-999"), + expected_field="region", + expected_check="region_code", + ), + Scenario( + id="division_area::admin_level_0:bounds", + scaffold={"admin_level": 0}, + mutate=set_at_path("admin_level", -1), + expected_field="admin_level_0", + expected_check="bounds", + ), + Scenario( + id="division_area::admin_level_1:bounds", + scaffold={"admin_level": 0}, + mutate=set_at_path("admin_level", 17), + expected_field="admin_level_1", + expected_check="bounds", + ), + Scenario( + id="division_area::model:require_any_true:0", + scaffold={}, + mutate=lambda row: mutate_require_any_true( + row, {"is_land": False, "is_territorial": False} + ), + expected_field="require_any_true", + expected_check="require_any_true", + ), + Scenario( + id="division_area::model:require_if:1", + scaffold={}, + mutate=lambda row: mutate_require_if(row, ["admin_level"], "subtype", "county"), + expected_field="admin_level_required_0", + expected_check="require_if", + ), + Scenario( + id="division_area::model:require_if:2", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["admin_level"], "subtype", "macrocounty" + ), + expected_field="admin_level_required_1", + expected_check="require_if", + ), + Scenario( + id="division_area::model:require_if:3", + scaffold={}, + mutate=lambda row: mutate_require_if(row, ["admin_level"], "subtype", "region"), + expected_field="admin_level_required_2", + expected_check="require_if", + ), + Scenario( + id="division_area::model:require_if:4", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["admin_level"], "subtype", "macroregion" + ), + expected_field="admin_level_required_3", + expected_check="require_if", + ), + Scenario( + id="division_area::model:require_if:5", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["admin_level"], "subtype", "dependency" + ), + expected_field="admin_level_required_4", + expected_check="require_if", + ), + Scenario( + id="division_area::model:require_if:6", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["admin_level"], "subtype", "country" + ), + expected_field="admin_level_required_5", + expected_check="require_if", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return division_area_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + DIVISION_AREA_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="division_area", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + DIVISION_AREA_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="division_area", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("division_area::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("division_area::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/divisions/test_division_boundary.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/divisions/test_division_boundary.py new file mode 100644 index 000000000..8f3e1de23 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/divisions/test_division_boundary.py @@ -0,0 +1,638 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for division_boundary.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.divisions.division_boundary import ( + DIVISION_BOUNDARY_SCHEMA, + division_boundary_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_forbid_if, + mutate_require_any_true, + mutate_require_if, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "3c9e8190-33ce-5962-9668-d467336901b4", + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "divisions", + "type": "division_boundary", + "version": 0, + "subtype": "country", + "class": "land", + "division_ids": ["a", "a1"], + "is_land": True, + "admin_level": 0, +} + + +BASE_ROW_POPULATED: dict = { + "id": "3c9e8190-33ce-5962-9668-d467336901b4", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "divisions", + "type": "division_boundary", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "subtype": "country", + "class": "land", + "is_land": True, + "is_territorial": False, + "division_ids": ["a", "a1"], + "region": "US-CA", + "admin_level": 0, + "is_disputed": False, + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="division_boundary::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="division_boundary::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="division_boundary::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="division_boundary::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="division_boundary::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="division_boundary::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="division_boundary::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="division_boundary::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "POINT (0 0)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="division_boundary::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="division_boundary::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="division_boundary::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="division_boundary::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="division_boundary::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="division_boundary::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="division_boundary::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="division_boundary::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="division_boundary::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="division_boundary::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="division_boundary::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="division_boundary::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="division_boundary::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="division_boundary::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="division_boundary::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="division_boundary::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="division_boundary::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="division_boundary::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="division_boundary::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="division_boundary::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="division_boundary::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="division_boundary::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="division_boundary::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="division_boundary::subtype:required", + scaffold={}, + mutate=set_at_path("subtype", None), + expected_field="subtype", + expected_check="required", + ), + Scenario( + id="division_boundary::subtype:enum", + scaffold={}, + mutate=set_at_path("subtype", "__INVALID__"), + expected_field="subtype", + expected_check="enum", + ), + Scenario( + id="division_boundary::class:required", + scaffold={}, + mutate=set_at_path("class", None), + expected_field="class", + expected_check="required", + ), + Scenario( + id="division_boundary::class:enum", + scaffold={}, + mutate=set_at_path("class", "__INVALID__"), + expected_field="class", + expected_check="enum", + ), + Scenario( + id="division_boundary::division_ids:required", + scaffold={}, + mutate=set_at_path("division_ids", None), + expected_field="division_ids", + expected_check="required", + ), + Scenario( + id="division_boundary::division_ids_min_length:array_min_length", + scaffold={}, + mutate=set_at_path("division_ids", []), + expected_field="division_ids_min_length", + expected_check="array_min_length", + ), + Scenario( + id="division_boundary::division_ids_max_length:array_max_length", + scaffold={}, + mutate=set_at_path("division_ids", [{}, {}, {}]), + expected_field="division_ids_max_length", + expected_check="array_max_length", + ), + Scenario( + id="division_boundary::division_ids_unique:struct_unique", + scaffold={}, + mutate=lambda row: mutate_unique_items(row, "division_ids"), + expected_field="division_ids_unique", + expected_check="struct_unique", + ), + Scenario( + id="division_boundary::division_ids[]:string_min_length", + scaffold={}, + mutate=set_at_path("division_ids[]", ""), + expected_field="division_ids[]", + expected_check="string_min_length", + ), + Scenario( + id="division_boundary::division_ids[]:no_whitespace", + scaffold={}, + mutate=set_at_path("division_ids[]", "has whitespace"), + expected_field="division_ids[]", + expected_check="no_whitespace", + ), + Scenario( + id="division_boundary::country:country_code_alpha2", + scaffold={"subtype": "dependency", "country": "US"}, + mutate=set_at_path("country", "99"), + expected_field="country", + expected_check="country_code_alpha2", + ), + Scenario( + id="division_boundary::region:region_code", + scaffold={"region": "US-CA"}, + mutate=set_at_path("region", "99-999"), + expected_field="region", + expected_check="region_code", + ), + Scenario( + id="division_boundary::admin_level_0:bounds", + scaffold={"admin_level": 0}, + mutate=set_at_path("admin_level", -1), + expected_field="admin_level_0", + expected_check="bounds", + ), + Scenario( + id="division_boundary::admin_level_1:bounds", + scaffold={"admin_level": 0}, + mutate=set_at_path("admin_level", 17), + expected_field="admin_level_1", + expected_check="bounds", + ), + Scenario( + id="division_boundary::perspectives.mode:required", + scaffold={"perspectives": {"mode": "accepted_by", "countries": ["US"]}}, + mutate=set_at_path("perspectives.mode", None), + expected_field="perspectives.mode", + expected_check="required", + ), + Scenario( + id="division_boundary::perspectives.mode:enum", + scaffold={"perspectives": {"mode": "accepted_by", "countries": ["US"]}}, + mutate=set_at_path("perspectives.mode", "__INVALID__"), + expected_field="perspectives.mode", + expected_check="enum", + ), + Scenario( + id="division_boundary::perspectives.countries:required", + scaffold={"perspectives": {"mode": "accepted_by", "countries": ["US"]}}, + mutate=set_at_path("perspectives.countries", None), + expected_field="perspectives.countries", + expected_check="required", + ), + Scenario( + id="division_boundary::perspectives.countries_min_length:array_min_length", + scaffold={"perspectives": {"mode": "accepted_by", "countries": ["US"]}}, + mutate=set_at_path("perspectives.countries", []), + expected_field="perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="division_boundary::perspectives.countries_unique:struct_unique", + scaffold={"perspectives": {"mode": "accepted_by", "countries": ["US"]}}, + mutate=lambda row: mutate_unique_items(row, "perspectives.countries"), + expected_field="perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="division_boundary::perspectives.countries[]:country_code_alpha2", + scaffold={"perspectives": {"mode": "accepted_by", "countries": ["US"]}}, + mutate=set_at_path("perspectives.countries[]", "99"), + expected_field="perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="division_boundary::model:require_any_true:0", + scaffold={}, + mutate=lambda row: mutate_require_any_true( + row, {"is_land": False, "is_territorial": False} + ), + expected_field="require_any_true", + expected_check="require_any_true", + ), + Scenario( + id="division_boundary::model:require_if:1", + scaffold={}, + mutate=lambda row: mutate_require_if(row, ["admin_level"], "subtype", "county"), + expected_field="admin_level_required_0", + expected_check="require_if", + ), + Scenario( + id="division_boundary::model:require_if:2", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["admin_level"], "subtype", "macrocounty" + ), + expected_field="admin_level_required_1", + expected_check="require_if", + ), + Scenario( + id="division_boundary::model:require_if:3", + scaffold={}, + mutate=lambda row: mutate_require_if(row, ["admin_level"], "subtype", "region"), + expected_field="admin_level_required_2", + expected_check="require_if", + ), + Scenario( + id="division_boundary::model:require_if:4", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["admin_level"], "subtype", "macroregion" + ), + expected_field="admin_level_required_3", + expected_check="require_if", + ), + Scenario( + id="division_boundary::model:require_if:5", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["admin_level"], "subtype", "dependency" + ), + expected_field="admin_level_required_4", + expected_check="require_if", + ), + Scenario( + id="division_boundary::model:require_if:6", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["admin_level"], "subtype", "country" + ), + expected_field="admin_level_required_5", + expected_check="require_if", + ), + Scenario( + id="division_boundary::model:require_if:7", + scaffold={}, + mutate=lambda row: mutate_require_if( + row, ["country"], "subtype", "country", negate=True + ), + expected_field="country_required", + expected_check="require_if", + ), + Scenario( + id="division_boundary::model:forbid_if:8", + scaffold={}, + mutate=lambda row: mutate_forbid_if(row, ["country"], "subtype", "country"), + expected_field="country_forbidden", + expected_check="forbid_if", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return division_boundary_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + DIVISION_BOUNDARY_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="division_boundary", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + DIVISION_BOUNDARY_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="division_boundary", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("division_boundary::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("division_boundary::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/places/__init__.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/places/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/places/test_place.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/places/test_place.py new file mode 100644 index 000000000..1e2b57db7 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/places/test_place.py @@ -0,0 +1,1301 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for place.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.places.place import ( + PLACE_SCHEMA, + place_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_map_key, + mutate_map_value, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "771dc733-3cd9-5ec4-a0b9-946ff01afb4e", + "geometry": "POINT (0 0)", + "theme": "places", + "type": "place", + "version": 0, +} + + +BASE_ROW_POPULATED: dict = { + "id": "771dc733-3cd9-5ec4-a0b9-946ff01afb4e", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "POINT (0 0)", + "theme": "places", + "type": "place", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "operating_status": "open", + "categories": {"primary": "snake_case", "alternate": ["snake_case"]}, + "basic_category": "snake_case", + "taxonomy": { + "primary": "snake_case", + "hierarchy": ["snake_case"], + "alternates": ["snake_case"], + }, + "confidence": 0.0, + "websites": ["https://example.com/"], + "socials": ["https://example.com/"], + "emails": ["user@example.com"], + "phones": ["+1 555-555-5555"], + "brand": { + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, + "wikidata": "Q42", + }, + "addresses": [ + { + "freeform": "", + "locality": "", + "postcode": "", + "region": "US-CA", + "country": "US", + } + ], + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="place::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="place::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="place::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="place::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="place::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="place::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="place::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="place::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "LINESTRING (0 0, 1 1)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="place::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="place::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="place::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="place::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="place::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="place::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="place::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="place::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="place::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="place::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="place::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="place::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="place::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="place::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="place::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="place::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="place::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="place::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="place::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="place::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="place::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="place::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="place::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="place::operating_status:enum", + scaffold={"operating_status": "open"}, + mutate=set_at_path("operating_status", "__INVALID__"), + expected_field="operating_status", + expected_check="enum", + ), + Scenario( + id="place::categories.primary:required", + scaffold={"categories": {"primary": "snake_case"}}, + mutate=set_at_path("categories.primary", None), + expected_field="categories.primary", + expected_check="required", + ), + Scenario( + id="place::categories.primary:snake_case", + scaffold={"categories": {"primary": "snake_case"}}, + mutate=set_at_path("categories.primary", "HAS SPACES"), + expected_field="categories.primary", + expected_check="snake_case", + ), + Scenario( + id="place::categories.alternate_unique:struct_unique", + scaffold={"categories": {"primary": "snake_case", "alternate": ["snake_case"]}}, + mutate=lambda row: mutate_unique_items(row, "categories.alternate"), + expected_field="categories.alternate_unique", + expected_check="struct_unique", + ), + Scenario( + id="place::categories.alternate[]:snake_case", + scaffold={"categories": {"primary": "snake_case", "alternate": ["snake_case"]}}, + mutate=set_at_path("categories.alternate[]", "HAS SPACES"), + expected_field="categories.alternate[]", + expected_check="snake_case", + ), + Scenario( + id="place::basic_category:snake_case", + scaffold={"basic_category": "snake_case"}, + mutate=set_at_path("basic_category", "HAS SPACES"), + expected_field="basic_category", + expected_check="snake_case", + ), + Scenario( + id="place::taxonomy.primary:required", + scaffold={"taxonomy": {"primary": "snake_case", "hierarchy": ["snake_case"]}}, + mutate=set_at_path("taxonomy.primary", None), + expected_field="taxonomy.primary", + expected_check="required", + ), + Scenario( + id="place::taxonomy.primary:snake_case", + scaffold={"taxonomy": {"primary": "snake_case", "hierarchy": ["snake_case"]}}, + mutate=set_at_path("taxonomy.primary", "HAS SPACES"), + expected_field="taxonomy.primary", + expected_check="snake_case", + ), + Scenario( + id="place::taxonomy.hierarchy:required", + scaffold={"taxonomy": {"primary": "snake_case", "hierarchy": ["snake_case"]}}, + mutate=set_at_path("taxonomy.hierarchy", None), + expected_field="taxonomy.hierarchy", + expected_check="required", + ), + Scenario( + id="place::taxonomy.hierarchy_min_length:array_min_length", + scaffold={"taxonomy": {"primary": "snake_case", "hierarchy": ["snake_case"]}}, + mutate=set_at_path("taxonomy.hierarchy", []), + expected_field="taxonomy.hierarchy_min_length", + expected_check="array_min_length", + ), + Scenario( + id="place::taxonomy.hierarchy_unique:struct_unique", + scaffold={"taxonomy": {"primary": "snake_case", "hierarchy": ["snake_case"]}}, + mutate=lambda row: mutate_unique_items(row, "taxonomy.hierarchy"), + expected_field="taxonomy.hierarchy_unique", + expected_check="struct_unique", + ), + Scenario( + id="place::taxonomy.hierarchy[]:snake_case", + scaffold={"taxonomy": {"primary": "snake_case", "hierarchy": ["snake_case"]}}, + mutate=set_at_path("taxonomy.hierarchy[]", "HAS SPACES"), + expected_field="taxonomy.hierarchy[]", + expected_check="snake_case", + ), + Scenario( + id="place::taxonomy.alternates_min_length:array_min_length", + scaffold={ + "taxonomy": { + "primary": "snake_case", + "hierarchy": ["snake_case"], + "alternates": ["snake_case"], + } + }, + mutate=set_at_path("taxonomy.alternates", []), + expected_field="taxonomy.alternates_min_length", + expected_check="array_min_length", + ), + Scenario( + id="place::taxonomy.alternates_unique:struct_unique", + scaffold={ + "taxonomy": { + "primary": "snake_case", + "hierarchy": ["snake_case"], + "alternates": ["snake_case"], + } + }, + mutate=lambda row: mutate_unique_items(row, "taxonomy.alternates"), + expected_field="taxonomy.alternates_unique", + expected_check="struct_unique", + ), + Scenario( + id="place::taxonomy.alternates[]:snake_case", + scaffold={ + "taxonomy": { + "primary": "snake_case", + "hierarchy": ["snake_case"], + "alternates": ["snake_case"], + } + }, + mutate=set_at_path("taxonomy.alternates[]", "HAS SPACES"), + expected_field="taxonomy.alternates[]", + expected_check="snake_case", + ), + Scenario( + id="place::confidence_0:bounds", + scaffold={"confidence": 0.0}, + mutate=set_at_path("confidence", -1.0), + expected_field="confidence_0", + expected_check="bounds", + ), + Scenario( + id="place::confidence_1:bounds", + scaffold={"confidence": 0.0}, + mutate=set_at_path("confidence", 2.0), + expected_field="confidence_1", + expected_check="bounds", + ), + Scenario( + id="place::websites_min_length:array_min_length", + scaffold={"websites": ["https://example.com/"]}, + mutate=set_at_path("websites", []), + expected_field="websites_min_length", + expected_check="array_min_length", + ), + Scenario( + id="place::websites_unique:struct_unique", + scaffold={"websites": ["https://example.com/"]}, + mutate=lambda row: mutate_unique_items(row, "websites"), + expected_field="websites_unique", + expected_check="struct_unique", + ), + Scenario( + id="place::websites[]:url_format", + scaffold={"websites": ["https://example.com/"]}, + mutate=set_at_path("websites[]", "not-a-url"), + expected_field="websites[]", + expected_check="url_format", + ), + Scenario( + id="place::websites[]:url_length", + scaffold={"websites": ["https://example.com/"]}, + mutate=set_at_path( + "websites[]", + "https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + ), + expected_field="websites[]", + expected_check="url_length", + ), + Scenario( + id="place::socials_min_length:array_min_length", + scaffold={"socials": ["https://example.com/"]}, + mutate=set_at_path("socials", []), + expected_field="socials_min_length", + expected_check="array_min_length", + ), + Scenario( + id="place::socials_unique:struct_unique", + scaffold={"socials": ["https://example.com/"]}, + mutate=lambda row: mutate_unique_items(row, "socials"), + expected_field="socials_unique", + expected_check="struct_unique", + ), + Scenario( + id="place::socials[]:url_format", + scaffold={"socials": ["https://example.com/"]}, + mutate=set_at_path("socials[]", "not-a-url"), + expected_field="socials[]", + expected_check="url_format", + ), + Scenario( + id="place::socials[]:url_length", + scaffold={"socials": ["https://example.com/"]}, + mutate=set_at_path( + "socials[]", + "https://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + ), + expected_field="socials[]", + expected_check="url_length", + ), + Scenario( + id="place::emails_min_length:array_min_length", + scaffold={"emails": ["user@example.com"]}, + mutate=set_at_path("emails", []), + expected_field="emails_min_length", + expected_check="array_min_length", + ), + Scenario( + id="place::emails_unique:struct_unique", + scaffold={"emails": ["user@example.com"]}, + mutate=lambda row: mutate_unique_items(row, "emails"), + expected_field="emails_unique", + expected_check="struct_unique", + ), + Scenario( + id="place::emails[]:email", + scaffold={"emails": ["user@example.com"]}, + mutate=set_at_path("emails[]", "not-an-email"), + expected_field="emails[]", + expected_check="email", + ), + Scenario( + id="place::phones_min_length:array_min_length", + scaffold={"phones": ["+1 555-555-5555"]}, + mutate=set_at_path("phones", []), + expected_field="phones_min_length", + expected_check="array_min_length", + ), + Scenario( + id="place::phones_unique:struct_unique", + scaffold={"phones": ["+1 555-555-5555"]}, + mutate=lambda row: mutate_unique_items(row, "phones"), + expected_field="phones_unique", + expected_check="struct_unique", + ), + Scenario( + id="place::phones[]:phone_number", + scaffold={"phones": ["+1 555-555-5555"]}, + mutate=set_at_path("phones[]", "1234567890"), + expected_field="phones[]", + expected_check="phone_number", + ), + Scenario( + id="place::brand.names.primary:required", + scaffold={"brand": {"names": {"primary": "a"}}}, + mutate=set_at_path("brand.names.primary", None), + expected_field="brand.names.primary", + expected_check="required", + ), + Scenario( + id="place::brand.names.primary:string_min_length", + scaffold={"brand": {"names": {"primary": "a"}}}, + mutate=set_at_path("brand.names.primary", ""), + expected_field="brand.names.primary", + expected_check="string_min_length", + ), + Scenario( + id="place::brand.names.primary:stripped", + scaffold={"brand": {"names": {"primary": "a"}}}, + mutate=set_at_path("brand.names.primary", " has spaces "), + expected_field="brand.names.primary", + expected_check="stripped", + ), + Scenario( + id="place::brand.names.common{key}:language_tag", + scaffold={"brand": {"names": {"primary": "a", "common": {"en": "clean"}}}}, + mutate=lambda row: mutate_map_key(row, "brand.names.common", "123"), + expected_field="brand.names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="place::brand.names.common{value}:stripped", + scaffold={"brand": {"names": {"primary": "a", "common": {"en": "clean"}}}}, + mutate=lambda row: mutate_map_value(row, "brand.names.common", " has spaces "), + expected_field="brand.names.common{value}", + expected_check="stripped", + ), + Scenario( + id="place::brand.names.rules[].value:required", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common"}], + } + } + }, + mutate=set_at_path("brand.names.rules[].value", None), + expected_field="brand.names.rules[].value", + expected_check="required", + ), + Scenario( + id="place::brand.names.rules[].value:string_min_length", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common"}], + } + } + }, + mutate=set_at_path("brand.names.rules[].value", ""), + expected_field="brand.names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="place::brand.names.rules[].value:stripped", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common"}], + } + } + }, + mutate=set_at_path("brand.names.rules[].value", " has spaces "), + expected_field="brand.names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="place::brand.names.rules[].variant:required", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common"}], + } + } + }, + mutate=set_at_path("brand.names.rules[].variant", None), + expected_field="brand.names.rules[].variant", + expected_check="required", + ), + Scenario( + id="place::brand.names.rules[].variant:enum", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common"}], + } + } + }, + mutate=set_at_path("brand.names.rules[].variant", "__INVALID__"), + expected_field="brand.names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="place::brand.names.rules[].language:language_tag", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + } + }, + mutate=set_at_path("brand.names.rules[].language", "123"), + expected_field="brand.names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="place::brand.names.rules[].perspectives.mode:required", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": { + "mode": "accepted_by", + "countries": ["US"], + }, + } + ], + } + } + }, + mutate=set_at_path("brand.names.rules[].perspectives.mode", None), + expected_field="brand.names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="place::brand.names.rules[].perspectives.mode:enum", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": { + "mode": "accepted_by", + "countries": ["US"], + }, + } + ], + } + } + }, + mutate=set_at_path("brand.names.rules[].perspectives.mode", "__INVALID__"), + expected_field="brand.names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="place::brand.names.rules[].perspectives.countries:required", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": { + "mode": "accepted_by", + "countries": ["US"], + }, + } + ], + } + } + }, + mutate=set_at_path("brand.names.rules[].perspectives.countries", None), + expected_field="brand.names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="place::brand.names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": { + "mode": "accepted_by", + "countries": ["US"], + }, + } + ], + } + } + }, + mutate=set_at_path("brand.names.rules[].perspectives.countries", []), + expected_field="brand.names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="place::brand.names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": { + "mode": "accepted_by", + "countries": ["US"], + }, + } + ], + } + } + }, + mutate=lambda row: mutate_unique_items( + row, "brand.names.rules[].perspectives.countries" + ), + expected_field="brand.names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="place::brand.names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": { + "mode": "accepted_by", + "countries": ["US"], + }, + } + ], + } + } + }, + mutate=set_at_path("brand.names.rules[].perspectives.countries[]", "99"), + expected_field="brand.names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="place::brand.names.rules[].between:linear_range_length", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [ + {"value": "a", "variant": "common", "between": [0.0, 1.0]} + ], + } + } + }, + mutate=set_at_path("brand.names.rules[].between", [0.5]), + expected_field="brand.names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="place::brand.names.rules[].between:linear_range_bounds", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [ + {"value": "a", "variant": "common", "between": [0.0, 1.0]} + ], + } + } + }, + mutate=set_at_path("brand.names.rules[].between", [1.5, 2.0]), + expected_field="brand.names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="place::brand.names.rules[].between:linear_range_order", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [ + {"value": "a", "variant": "common", "between": [0.0, 1.0]} + ], + } + } + }, + mutate=set_at_path("brand.names.rules[].between", [0.8, 0.2]), + expected_field="brand.names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="place::brand.names.rules[].side:enum", + scaffold={ + "brand": { + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + } + }, + mutate=set_at_path("brand.names.rules[].side", "__INVALID__"), + expected_field="brand.names.rules[].side", + expected_check="enum", + ), + Scenario( + id="place::brand.wikidata:wikidata_id", + scaffold={"brand": {"wikidata": "Q42"}}, + mutate=set_at_path("brand.wikidata", "P999"), + expected_field="brand.wikidata", + expected_check="wikidata_id", + ), + Scenario( + id="place::addresses_min_length:array_min_length", + scaffold={"addresses": [{}]}, + mutate=set_at_path("addresses", []), + expected_field="addresses_min_length", + expected_check="array_min_length", + ), + Scenario( + id="place::addresses[].region:region_code", + scaffold={"addresses": [{"region": "US-CA"}]}, + mutate=set_at_path("addresses[].region", "99-999"), + expected_field="addresses[].region", + expected_check="region_code", + ), + Scenario( + id="place::addresses[].country:country_code_alpha2", + scaffold={"addresses": [{"country": "US"}]}, + mutate=set_at_path("addresses[].country", "99"), + expected_field="addresses[].country", + expected_check="country_code_alpha2", + ), + Scenario( + id="place::names.primary:required", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", None), + expected_field="names.primary", + expected_check="required", + ), + Scenario( + id="place::names.primary:string_min_length", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", ""), + expected_field="names.primary", + expected_check="string_min_length", + ), + Scenario( + id="place::names.primary:stripped", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", " has spaces "), + expected_field="names.primary", + expected_check="stripped", + ), + Scenario( + id="place::names.common{key}:language_tag", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_key(row, "names.common", "123"), + expected_field="names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="place::names.common{value}:stripped", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_value(row, "names.common", " has spaces "), + expected_field="names.common{value}", + expected_check="stripped", + ), + Scenario( + id="place::names.rules[].value:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", None), + expected_field="names.rules[].value", + expected_check="required", + ), + Scenario( + id="place::names.rules[].value:string_min_length", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", ""), + expected_field="names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="place::names.rules[].value:stripped", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", " has spaces "), + expected_field="names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="place::names.rules[].variant:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", None), + expected_field="names.rules[].variant", + expected_check="required", + ), + Scenario( + id="place::names.rules[].variant:enum", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", "__INVALID__"), + expected_field="names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="place::names.rules[].language:language_tag", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + }, + mutate=set_at_path("names.rules[].language", "123"), + expected_field="names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="place::names.rules[].perspectives.mode:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", None), + expected_field="names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="place::names.rules[].perspectives.mode:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", "__INVALID__"), + expected_field="names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="place::names.rules[].perspectives.countries:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", None), + expected_field="names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="place::names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", []), + expected_field="names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="place::names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=lambda row: mutate_unique_items( + row, "names.rules[].perspectives.countries" + ), + expected_field="names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="place::names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries[]", "99"), + expected_field="names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="place::names.rules[].between:linear_range_length", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.5]), + expected_field="names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="place::names.rules[].between:linear_range_bounds", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [1.5, 2.0]), + expected_field="names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="place::names.rules[].between:linear_range_order", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.8, 0.2]), + expected_field="names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="place::names.rules[].side:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + }, + mutate=set_at_path("names.rules[].side", "__INVALID__"), + expected_field="names.rules[].side", + expected_check="enum", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return place_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + PLACE_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="place", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + PLACE_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="place", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("place::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("place::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/__init__.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_connector.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_connector.py new file mode 100644 index 000000000..a94cc383a --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_connector.py @@ -0,0 +1,404 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for connector.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.transportation.connector import ( + CONNECTOR_SCHEMA, + connector_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import mutate_unique_items +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "257724e0-9751-53b0-9891-95a9ffa523da", + "geometry": "POINT (0 0)", + "theme": "transportation", + "type": "connector", + "version": 0, +} + + +BASE_ROW_POPULATED: dict = { + "id": "257724e0-9751-53b0-9891-95a9ffa523da", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "POINT (0 0)", + "theme": "transportation", + "type": "connector", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="connector::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="connector::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="connector::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="connector::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="connector::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="connector::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="connector::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="connector::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "LINESTRING (0 0, 1 1)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="connector::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="connector::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="connector::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="connector::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="connector::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="connector::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="connector::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="connector::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="connector::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="connector::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="connector::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="connector::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="connector::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="connector::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="connector::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="connector::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="connector::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="connector::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="connector::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="connector::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="connector::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="connector::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="connector::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return connector_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + CONNECTOR_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="connector", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + CONNECTOR_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="connector", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("connector::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("connector::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_rail.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_rail.py new file mode 100644 index 000000000..a59960996 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_rail.py @@ -0,0 +1,1815 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for segment.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.transportation.segment import ( + SEGMENT_SCHEMA, + segment_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_forbid_if, + mutate_map_key, + mutate_map_value, + mutate_require_any_of, + mutate_require_if, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "1f4d65c9-e092-52c4-b002-7c11ce69a554", + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "transportation", + "type": "segment", + "version": 0, + "subtype": "rail", + "class": "funicular", +} + + +BASE_ROW_POPULATED: dict = { + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, + "id": "1f4d65c9-e092-52c4-b002-7c11ce69a554", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "transportation", + "type": "segment", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "subtype": "rail", + "access_restrictions": [ + { + "access_type": "allowed", + "between": [0.0, 1.0], + "when": { + "heading": "forward", + "during": "", + "mode": ["vehicle"], + "using": ["as_customer"], + "recognized": ["as_permitted"], + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ], + "connectors": [{"connector_id": "a", "at": 0.0}, {"connector_id": "a1", "at": 0.0}], + "level_rules": [{"value": 0, "between": [0.0, 1.0]}], + "routes": [ + { + "name": "a", + "network": "a", + "ref": "a", + "symbol": "a", + "wikidata": "Q42", + "between": [0.0, 1.0], + } + ], + "subclass_rules": [{"value": "link", "between": [0.0, 1.0]}], + "class": "funicular", + "rail_flags": [{"values": ["is_bridge"], "between": [0.0, 1.0]}], +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="segment::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="segment::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="segment::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="segment::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="segment::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="segment::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="segment::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="segment::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "POINT (0 0)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="segment::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="segment::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="segment::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="segment::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="segment::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="segment::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="segment::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="segment::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="segment::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="segment::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="segment::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="segment::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="segment::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="segment::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="segment::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="segment::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="segment::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="segment::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="segment::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::subtype:required", + scaffold={}, + mutate=set_at_path("subtype", None), + expected_field="subtype", + expected_check="required", + ), + Scenario( + id="segment::subtype:enum", + scaffold={}, + mutate=set_at_path("subtype", "__INVALID__"), + expected_field="subtype", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions_min_length:array_min_length", + scaffold={"access_restrictions": [{"access_type": "allowed"}]}, + mutate=set_at_path("access_restrictions", []), + expected_field="access_restrictions_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions_unique:struct_unique", + scaffold={"access_restrictions": [{"access_type": "allowed"}]}, + mutate=lambda row: mutate_unique_items(row, "access_restrictions"), + expected_field="access_restrictions_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].access_type:required", + scaffold={"access_restrictions": [{"access_type": "allowed"}]}, + mutate=set_at_path("access_restrictions[].access_type", None), + expected_field="access_restrictions[].access_type", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].access_type:enum", + scaffold={"access_restrictions": [{"access_type": "allowed"}]}, + mutate=set_at_path("access_restrictions[].access_type", "__INVALID__"), + expected_field="access_restrictions[].access_type", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].between:linear_range_length", + scaffold={ + "access_restrictions": [{"access_type": "allowed", "between": [0.0, 1.0]}] + }, + mutate=set_at_path("access_restrictions[].between", [0.5]), + expected_field="access_restrictions[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::access_restrictions[].between:linear_range_bounds", + scaffold={ + "access_restrictions": [{"access_type": "allowed", "between": [0.0, 1.0]}] + }, + mutate=set_at_path("access_restrictions[].between", [1.5, 2.0]), + expected_field="access_restrictions[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::access_restrictions[].between:linear_range_order", + scaffold={ + "access_restrictions": [{"access_type": "allowed", "between": [0.0, 1.0]}] + }, + mutate=set_at_path("access_restrictions[].between", [0.8, 0.2]), + expected_field="access_restrictions[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::access_restrictions[].when.heading:enum", + scaffold={ + "access_restrictions": [ + {"access_type": "allowed", "when": {"heading": "forward"}} + ] + }, + mutate=set_at_path("access_restrictions[].when.heading", "__INVALID__"), + expected_field="access_restrictions[].when.heading", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.mode_min_length:array_min_length", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.mode", []), + expected_field="access_restrictions[].when.mode_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions[].when.mode_unique:struct_unique", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items(row, "access_restrictions[].when.mode"), + expected_field="access_restrictions[].when.mode_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].when.mode[]:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.mode[]", "__INVALID__"), + expected_field="access_restrictions[].when.mode[]", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.using_min_length:array_min_length", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.using", []), + expected_field="access_restrictions[].when.using_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions[].when.using_unique:struct_unique", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items(row, "access_restrictions[].when.using"), + expected_field="access_restrictions[].when.using_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].when.using[]:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.using[]", "__INVALID__"), + expected_field="access_restrictions[].when.using[]", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.recognized_min_length:array_min_length", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.recognized", []), + expected_field="access_restrictions[].when.recognized_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions[].when.recognized_unique:struct_unique", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items( + row, "access_restrictions[].when.recognized" + ), + expected_field="access_restrictions[].when.recognized_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].when.recognized[]:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.recognized[]", "__INVALID__"), + expected_field="access_restrictions[].when.recognized[]", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle_min_length:array_min_length", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle", []), + expected_field="access_restrictions[].when.vehicle_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle_unique:struct_unique", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_unique_items( + row, "access_restrictions[].when.vehicle" + ), + expected_field="access_restrictions[].when.vehicle_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].dimension:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].dimension", None), + expected_field="access_restrictions[].when.vehicle[].dimension", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].dimension:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path( + "access_restrictions[].when.vehicle[].dimension", "__INVALID__" + ), + expected_field="access_restrictions[].when.vehicle[].dimension", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].comparison:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].comparison", None), + expected_field="access_restrictions[].when.vehicle[].comparison", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].comparison:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path( + "access_restrictions[].when.vehicle[].comparison", "__INVALID__" + ), + expected_field="access_restrictions[].when.vehicle[].comparison", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value_0:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 0, + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", None), + expected_field="access_restrictions[].when.vehicle[].value_0", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value_1:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", None), + expected_field="access_restrictions[].when.vehicle[].value_1", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value:bounds", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", -1.0), + expected_field="access_restrictions[].when.vehicle[].value", + expected_check="bounds", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].unit_0:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].unit", None), + expected_field="access_restrictions[].when.vehicle[].unit_0", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].unit_0:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].unit", "__INVALID__"), + expected_field="access_restrictions[].when.vehicle[].unit_0", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].unit_1:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "weight", + "comparison": "greater_than", + "value": 0.0, + "unit": "oz", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].unit", None), + expected_field="access_restrictions[].when.vehicle[].unit_1", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].unit_1:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "weight", + "comparison": "greater_than", + "value": 0.0, + "unit": "oz", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].unit", "__INVALID__"), + expected_field="access_restrictions[].when.vehicle[].unit_1", + expected_check="enum", + ), + Scenario( + id="segment::connectors_min_length:array_min_length", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=set_at_path("connectors", []), + expected_field="connectors_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::connectors_unique:struct_unique", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=lambda row: mutate_unique_items(row, "connectors"), + expected_field="connectors_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::connectors[].connector_id:required", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=set_at_path("connectors[].connector_id", None), + expected_field="connectors[].connector_id", + expected_check="required", + ), + Scenario( + id="segment::connectors[].connector_id:string_min_length", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=set_at_path("connectors[].connector_id", ""), + expected_field="connectors[].connector_id", + expected_check="string_min_length", + ), + Scenario( + id="segment::connectors[].connector_id:no_whitespace", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=set_at_path("connectors[].connector_id", "has whitespace"), + expected_field="connectors[].connector_id", + expected_check="no_whitespace", + ), + Scenario( + id="segment::connectors[].at_0:bounds", + scaffold={ + "connectors": [{"connector_id": "a", "at": 0.0}, {"connector_id": "a1"}] + }, + mutate=set_at_path("connectors[].at", -1.0), + expected_field="connectors[].at_0", + expected_check="bounds", + ), + Scenario( + id="segment::connectors[].at_1:bounds", + scaffold={ + "connectors": [{"connector_id": "a", "at": 0.0}, {"connector_id": "a1"}] + }, + mutate=set_at_path("connectors[].at", 2.0), + expected_field="connectors[].at_1", + expected_check="bounds", + ), + Scenario( + id="segment::level_rules[].value:required", + scaffold={"level_rules": [{"value": 0}]}, + mutate=set_at_path("level_rules[].value", None), + expected_field="level_rules[].value", + expected_check="required", + ), + Scenario( + id="segment::level_rules[].between:linear_range_length", + scaffold={"level_rules": [{"value": 0, "between": [0.0, 1.0]}]}, + mutate=set_at_path("level_rules[].between", [0.5]), + expected_field="level_rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::level_rules[].between:linear_range_bounds", + scaffold={"level_rules": [{"value": 0, "between": [0.0, 1.0]}]}, + mutate=set_at_path("level_rules[].between", [1.5, 2.0]), + expected_field="level_rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::level_rules[].between:linear_range_order", + scaffold={"level_rules": [{"value": 0, "between": [0.0, 1.0]}]}, + mutate=set_at_path("level_rules[].between", [0.8, 0.2]), + expected_field="level_rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::routes[].name:string_min_length", + scaffold={"routes": [{"name": "a"}]}, + mutate=set_at_path("routes[].name", ""), + expected_field="routes[].name", + expected_check="string_min_length", + ), + Scenario( + id="segment::routes[].name:stripped", + scaffold={"routes": [{"name": "a"}]}, + mutate=set_at_path("routes[].name", " has spaces "), + expected_field="routes[].name", + expected_check="stripped", + ), + Scenario( + id="segment::routes[].network:string_min_length", + scaffold={"routes": [{"network": "a"}]}, + mutate=set_at_path("routes[].network", ""), + expected_field="routes[].network", + expected_check="string_min_length", + ), + Scenario( + id="segment::routes[].network:stripped", + scaffold={"routes": [{"network": "a"}]}, + mutate=set_at_path("routes[].network", " has spaces "), + expected_field="routes[].network", + expected_check="stripped", + ), + Scenario( + id="segment::routes[].ref:string_min_length", + scaffold={"routes": [{"ref": "a"}]}, + mutate=set_at_path("routes[].ref", ""), + expected_field="routes[].ref", + expected_check="string_min_length", + ), + Scenario( + id="segment::routes[].ref:stripped", + scaffold={"routes": [{"ref": "a"}]}, + mutate=set_at_path("routes[].ref", " has spaces "), + expected_field="routes[].ref", + expected_check="stripped", + ), + Scenario( + id="segment::routes[].symbol:string_min_length", + scaffold={"routes": [{"symbol": "a"}]}, + mutate=set_at_path("routes[].symbol", ""), + expected_field="routes[].symbol", + expected_check="string_min_length", + ), + Scenario( + id="segment::routes[].symbol:stripped", + scaffold={"routes": [{"symbol": "a"}]}, + mutate=set_at_path("routes[].symbol", " has spaces "), + expected_field="routes[].symbol", + expected_check="stripped", + ), + Scenario( + id="segment::routes[].wikidata:wikidata_id", + scaffold={"routes": [{"wikidata": "Q42"}]}, + mutate=set_at_path("routes[].wikidata", "P999"), + expected_field="routes[].wikidata", + expected_check="wikidata_id", + ), + Scenario( + id="segment::routes[].between:linear_range_length", + scaffold={"routes": [{"between": [0.0, 1.0]}]}, + mutate=set_at_path("routes[].between", [0.5]), + expected_field="routes[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::routes[].between:linear_range_bounds", + scaffold={"routes": [{"between": [0.0, 1.0]}]}, + mutate=set_at_path("routes[].between", [1.5, 2.0]), + expected_field="routes[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::routes[].between:linear_range_order", + scaffold={"routes": [{"between": [0.0, 1.0]}]}, + mutate=set_at_path("routes[].between", [0.8, 0.2]), + expected_field="routes[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::subclass_rules[].value:required", + scaffold={"subclass_rules": [{"value": "link"}]}, + mutate=set_at_path("subclass_rules[].value", None), + expected_field="subclass_rules[].value", + expected_check="required", + ), + Scenario( + id="segment::subclass_rules[].value:enum", + scaffold={"subclass_rules": [{"value": "link"}]}, + mutate=set_at_path("subclass_rules[].value", "__INVALID__"), + expected_field="subclass_rules[].value", + expected_check="enum", + ), + Scenario( + id="segment::subclass_rules[].between:linear_range_length", + scaffold={"subclass_rules": [{"value": "link", "between": [0.0, 1.0]}]}, + mutate=set_at_path("subclass_rules[].between", [0.5]), + expected_field="subclass_rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::subclass_rules[].between:linear_range_bounds", + scaffold={"subclass_rules": [{"value": "link", "between": [0.0, 1.0]}]}, + mutate=set_at_path("subclass_rules[].between", [1.5, 2.0]), + expected_field="subclass_rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::subclass_rules[].between:linear_range_order", + scaffold={"subclass_rules": [{"value": "link", "between": [0.0, 1.0]}]}, + mutate=set_at_path("subclass_rules[].between", [0.8, 0.2]), + expected_field="subclass_rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::names.primary:required", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", None), + expected_field="names.primary", + expected_check="required", + ), + Scenario( + id="segment::names.primary:string_min_length", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", ""), + expected_field="names.primary", + expected_check="string_min_length", + ), + Scenario( + id="segment::names.primary:stripped", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", " has spaces "), + expected_field="names.primary", + expected_check="stripped", + ), + Scenario( + id="segment::names.common{key}:language_tag", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_key(row, "names.common", "123"), + expected_field="names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="segment::names.common{value}:stripped", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_value(row, "names.common", " has spaces "), + expected_field="names.common{value}", + expected_check="stripped", + ), + Scenario( + id="segment::names.rules[].value:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", None), + expected_field="names.rules[].value", + expected_check="required", + ), + Scenario( + id="segment::names.rules[].value:string_min_length", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", ""), + expected_field="names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="segment::names.rules[].value:stripped", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", " has spaces "), + expected_field="names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="segment::names.rules[].variant:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", None), + expected_field="names.rules[].variant", + expected_check="required", + ), + Scenario( + id="segment::names.rules[].variant:enum", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", "__INVALID__"), + expected_field="names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="segment::names.rules[].language:language_tag", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + }, + mutate=set_at_path("names.rules[].language", "123"), + expected_field="names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="segment::names.rules[].perspectives.mode:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", None), + expected_field="names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="segment::names.rules[].perspectives.mode:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", "__INVALID__"), + expected_field="names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="segment::names.rules[].perspectives.countries:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", None), + expected_field="names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="segment::names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", []), + expected_field="names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=lambda row: mutate_unique_items( + row, "names.rules[].perspectives.countries" + ), + expected_field="names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries[]", "99"), + expected_field="names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="segment::names.rules[].between:linear_range_length", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.5]), + expected_field="names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::names.rules[].between:linear_range_bounds", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [1.5, 2.0]), + expected_field="names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::names.rules[].between:linear_range_order", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.8, 0.2]), + expected_field="names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::names.rules[].side:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + }, + mutate=set_at_path("names.rules[].side", "__INVALID__"), + expected_field="names.rules[].side", + expected_check="enum", + ), + Scenario( + id="segment::class_1:required", + scaffold={}, + mutate=set_at_path("class", None), + expected_field="class_1", + expected_check="required", + ), + Scenario( + id="segment::class_1:enum", + scaffold={}, + mutate=set_at_path("class", "__INVALID__"), + expected_field="class_1", + expected_check="enum", + ), + Scenario( + id="segment::rail_flags_min_length:array_min_length", + scaffold={"rail_flags": [{"values": ["is_bridge"]}]}, + mutate=set_at_path("rail_flags", []), + expected_field="rail_flags_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::rail_flags_unique:struct_unique", + scaffold={"rail_flags": [{"values": ["is_bridge"]}]}, + mutate=lambda row: mutate_unique_items(row, "rail_flags"), + expected_field="rail_flags_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::rail_flags[].values:required", + scaffold={"rail_flags": [{"values": ["is_bridge"]}]}, + mutate=set_at_path("rail_flags[].values", None), + expected_field="rail_flags[].values", + expected_check="required", + ), + Scenario( + id="segment::rail_flags[].values_min_length:array_min_length", + scaffold={"rail_flags": [{"values": ["is_bridge"]}]}, + mutate=set_at_path("rail_flags[].values", []), + expected_field="rail_flags[].values_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::rail_flags[].values_unique:struct_unique", + scaffold={"rail_flags": [{"values": ["is_bridge"]}]}, + mutate=lambda row: mutate_unique_items(row, "rail_flags[].values"), + expected_field="rail_flags[].values_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::rail_flags[].values[]:enum", + scaffold={"rail_flags": [{"values": ["is_bridge"]}]}, + mutate=set_at_path("rail_flags[].values[]", "__INVALID__"), + expected_field="rail_flags[].values[]", + expected_check="enum", + ), + Scenario( + id="segment::rail_flags[].between:linear_range_length", + scaffold={"rail_flags": [{"values": ["is_bridge"], "between": [0.0, 1.0]}]}, + mutate=set_at_path("rail_flags[].between", [0.5]), + expected_field="rail_flags[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::rail_flags[].between:linear_range_bounds", + scaffold={"rail_flags": [{"values": ["is_bridge"], "between": [0.0, 1.0]}]}, + mutate=set_at_path("rail_flags[].between", [1.5, 2.0]), + expected_field="rail_flags[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::rail_flags[].between:linear_range_order", + scaffold={"rail_flags": [{"values": ["is_bridge"], "between": [0.0, 1.0]}]}, + mutate=set_at_path("rail_flags[].between", [0.8, 0.2]), + expected_field="rail_flags[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::model:forbid_if:0", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_forbid_if( + row, + ["unit"], + "dimension", + "axle_count", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:require_if:1", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "height", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_required_0", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:2", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "length", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_required_1", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:3", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "weight", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_required_2", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:4", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "width", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_required_3", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_any_of:5", + scaffold={ + "access_restrictions": [ + {"access_type": "allowed", "when": {"heading": "forward"}} + ] + }, + mutate=lambda row: mutate_require_any_of( + row, + ["heading", "during", "mode", "using", "recognized", "vehicle"], + array_path="access_restrictions", + struct_path="when", + ), + expected_field="access_restrictions[].when", + expected_check="require_any_of", + ), + Scenario( + id="segment::model:forbid_if:6", + scaffold={}, + mutate=lambda row: mutate_forbid_if(row, ["class"], "subtype", "water"), + expected_field="class_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:require_if:7", + scaffold={}, + mutate=lambda row: mutate_require_if(row, ["class"], "subtype", "rail"), + expected_field="class_required_0", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:8", + scaffold={}, + mutate=lambda row: mutate_require_if(row, ["class"], "subtype", "road"), + expected_field="class_required_1", + expected_check="require_if", + ), + Scenario( + id="segment::model:forbid_if:9", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["destinations"], + "subtype", + "road", + negate=True, + fill_values={"destinations": [{}]}, + ), + expected_field="destinations_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:10", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["prohibited_transitions"], + "subtype", + "road", + negate=True, + fill_values={"prohibited_transitions": [{}]}, + ), + expected_field="prohibited_transitions_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:11", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["road_flags"], + "subtype", + "road", + negate=True, + fill_values={"road_flags": [{}]}, + ), + expected_field="road_flags_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:12", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["road_surface"], + "subtype", + "road", + negate=True, + fill_values={"road_surface": [{}]}, + ), + expected_field="road_surface_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:13", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["speed_limits"], + "subtype", + "road", + negate=True, + fill_values={"speed_limits": [{}]}, + ), + expected_field="speed_limits_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:14", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, ["subclass"], "subtype", "road", negate=True + ), + expected_field="subclass_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:15", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["width_rules"], + "subtype", + "road", + negate=True, + fill_values={"width_rules": [{}]}, + ), + expected_field="width_rules_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:16", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["rail_flags"], + "subtype", + "rail", + negate=True, + fill_values={"rail_flags": [{}]}, + ), + expected_field="rail_flags_forbidden", + expected_check="forbid_if", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return segment_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + SEGMENT_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="segment", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + SEGMENT_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="segment", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("segment::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("segment::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_road.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_road.py new file mode 100644 index 000000000..2e58fcda5 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_road.py @@ -0,0 +1,4037 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for segment.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.transportation.segment import ( + SEGMENT_SCHEMA, + segment_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_forbid_if, + mutate_map_key, + mutate_map_value, + mutate_require_any_of, + mutate_require_if, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "1f4d65c9-e092-52c4-b002-7c11ce69a554", + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "transportation", + "type": "segment", + "version": 0, + "subtype": "road", + "class": "motorway", +} + + +BASE_ROW_POPULATED: dict = { + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, + "id": "1f4d65c9-e092-52c4-b002-7c11ce69a554", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "transportation", + "type": "segment", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "subtype": "road", + "access_restrictions": [ + { + "access_type": "allowed", + "between": [0.0, 1.0], + "when": { + "heading": "forward", + "during": "", + "mode": ["vehicle"], + "using": ["as_customer"], + "recognized": ["as_permitted"], + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ], + "connectors": [{"connector_id": "a", "at": 0.0}, {"connector_id": "a1", "at": 0.0}], + "level_rules": [{"value": 0, "between": [0.0, 1.0]}], + "routes": [ + { + "name": "a", + "network": "a", + "ref": "a", + "symbol": "a", + "wikidata": "Q42", + "between": [0.0, 1.0], + } + ], + "subclass_rules": [{"value": "link", "between": [0.0, 1.0]}], + "class": "motorway", + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + "symbols": ["motorway"], + "when": {"heading": "forward"}, + } + ], + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "between": [0.0, 1.0], + "when": { + "heading": "forward", + "during": "", + "mode": ["vehicle"], + "using": ["as_customer"], + "recognized": ["as_permitted"], + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ], + "road_flags": [{"values": ["is_bridge"], "between": [0.0, 1.0]}], + "road_surface": [{"value": "unknown", "between": [0.0, 1.0]}], + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "min_speed": {"value": 1, "unit": "mph"}, + "is_max_speed_variable": False, + "between": [0.0, 1.0], + "when": { + "heading": "forward", + "during": "", + "mode": ["vehicle"], + "using": ["as_customer"], + "recognized": ["as_permitted"], + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ], + "subclass": "link", + "width_rules": [{"value": 1.0, "between": [0.0, 1.0]}], +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="segment::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="segment::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="segment::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="segment::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="segment::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="segment::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="segment::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="segment::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "POINT (0 0)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="segment::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="segment::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="segment::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="segment::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="segment::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="segment::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="segment::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="segment::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="segment::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="segment::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="segment::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="segment::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="segment::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="segment::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="segment::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="segment::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="segment::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="segment::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="segment::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::subtype:required", + scaffold={}, + mutate=set_at_path("subtype", None), + expected_field="subtype", + expected_check="required", + ), + Scenario( + id="segment::subtype:enum", + scaffold={}, + mutate=set_at_path("subtype", "__INVALID__"), + expected_field="subtype", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions_min_length:array_min_length", + scaffold={"access_restrictions": [{"access_type": "allowed"}]}, + mutate=set_at_path("access_restrictions", []), + expected_field="access_restrictions_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions_unique:struct_unique", + scaffold={"access_restrictions": [{"access_type": "allowed"}]}, + mutate=lambda row: mutate_unique_items(row, "access_restrictions"), + expected_field="access_restrictions_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].access_type:required", + scaffold={"access_restrictions": [{"access_type": "allowed"}]}, + mutate=set_at_path("access_restrictions[].access_type", None), + expected_field="access_restrictions[].access_type", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].access_type:enum", + scaffold={"access_restrictions": [{"access_type": "allowed"}]}, + mutate=set_at_path("access_restrictions[].access_type", "__INVALID__"), + expected_field="access_restrictions[].access_type", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].between:linear_range_length", + scaffold={ + "access_restrictions": [{"access_type": "allowed", "between": [0.0, 1.0]}] + }, + mutate=set_at_path("access_restrictions[].between", [0.5]), + expected_field="access_restrictions[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::access_restrictions[].between:linear_range_bounds", + scaffold={ + "access_restrictions": [{"access_type": "allowed", "between": [0.0, 1.0]}] + }, + mutate=set_at_path("access_restrictions[].between", [1.5, 2.0]), + expected_field="access_restrictions[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::access_restrictions[].between:linear_range_order", + scaffold={ + "access_restrictions": [{"access_type": "allowed", "between": [0.0, 1.0]}] + }, + mutate=set_at_path("access_restrictions[].between", [0.8, 0.2]), + expected_field="access_restrictions[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::access_restrictions[].when.heading:enum", + scaffold={ + "access_restrictions": [ + {"access_type": "allowed", "when": {"heading": "forward"}} + ] + }, + mutate=set_at_path("access_restrictions[].when.heading", "__INVALID__"), + expected_field="access_restrictions[].when.heading", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.mode_min_length:array_min_length", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.mode", []), + expected_field="access_restrictions[].when.mode_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions[].when.mode_unique:struct_unique", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items(row, "access_restrictions[].when.mode"), + expected_field="access_restrictions[].when.mode_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].when.mode[]:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.mode[]", "__INVALID__"), + expected_field="access_restrictions[].when.mode[]", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.using_min_length:array_min_length", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.using", []), + expected_field="access_restrictions[].when.using_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions[].when.using_unique:struct_unique", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items(row, "access_restrictions[].when.using"), + expected_field="access_restrictions[].when.using_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].when.using[]:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.using[]", "__INVALID__"), + expected_field="access_restrictions[].when.using[]", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.recognized_min_length:array_min_length", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.recognized", []), + expected_field="access_restrictions[].when.recognized_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions[].when.recognized_unique:struct_unique", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items( + row, "access_restrictions[].when.recognized" + ), + expected_field="access_restrictions[].when.recognized_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].when.recognized[]:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.recognized[]", "__INVALID__"), + expected_field="access_restrictions[].when.recognized[]", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle_min_length:array_min_length", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle", []), + expected_field="access_restrictions[].when.vehicle_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle_unique:struct_unique", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_unique_items( + row, "access_restrictions[].when.vehicle" + ), + expected_field="access_restrictions[].when.vehicle_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].dimension:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].dimension", None), + expected_field="access_restrictions[].when.vehicle[].dimension", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].dimension:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path( + "access_restrictions[].when.vehicle[].dimension", "__INVALID__" + ), + expected_field="access_restrictions[].when.vehicle[].dimension", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].comparison:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].comparison", None), + expected_field="access_restrictions[].when.vehicle[].comparison", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].comparison:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path( + "access_restrictions[].when.vehicle[].comparison", "__INVALID__" + ), + expected_field="access_restrictions[].when.vehicle[].comparison", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value_0:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 0, + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", None), + expected_field="access_restrictions[].when.vehicle[].value_0", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value_1:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", None), + expected_field="access_restrictions[].when.vehicle[].value_1", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value:bounds", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", -1.0), + expected_field="access_restrictions[].when.vehicle[].value", + expected_check="bounds", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].unit_0:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].unit", None), + expected_field="access_restrictions[].when.vehicle[].unit_0", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].unit_0:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].unit", "__INVALID__"), + expected_field="access_restrictions[].when.vehicle[].unit_0", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].unit_1:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "weight", + "comparison": "greater_than", + "value": 0.0, + "unit": "oz", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].unit", None), + expected_field="access_restrictions[].when.vehicle[].unit_1", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].unit_1:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "weight", + "comparison": "greater_than", + "value": 0.0, + "unit": "oz", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].unit", "__INVALID__"), + expected_field="access_restrictions[].when.vehicle[].unit_1", + expected_check="enum", + ), + Scenario( + id="segment::connectors_min_length:array_min_length", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=set_at_path("connectors", []), + expected_field="connectors_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::connectors_unique:struct_unique", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=lambda row: mutate_unique_items(row, "connectors"), + expected_field="connectors_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::connectors[].connector_id:required", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=set_at_path("connectors[].connector_id", None), + expected_field="connectors[].connector_id", + expected_check="required", + ), + Scenario( + id="segment::connectors[].connector_id:string_min_length", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=set_at_path("connectors[].connector_id", ""), + expected_field="connectors[].connector_id", + expected_check="string_min_length", + ), + Scenario( + id="segment::connectors[].connector_id:no_whitespace", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=set_at_path("connectors[].connector_id", "has whitespace"), + expected_field="connectors[].connector_id", + expected_check="no_whitespace", + ), + Scenario( + id="segment::connectors[].at_0:bounds", + scaffold={ + "connectors": [{"connector_id": "a", "at": 0.0}, {"connector_id": "a1"}] + }, + mutate=set_at_path("connectors[].at", -1.0), + expected_field="connectors[].at_0", + expected_check="bounds", + ), + Scenario( + id="segment::connectors[].at_1:bounds", + scaffold={ + "connectors": [{"connector_id": "a", "at": 0.0}, {"connector_id": "a1"}] + }, + mutate=set_at_path("connectors[].at", 2.0), + expected_field="connectors[].at_1", + expected_check="bounds", + ), + Scenario( + id="segment::level_rules[].value:required", + scaffold={"level_rules": [{"value": 0}]}, + mutate=set_at_path("level_rules[].value", None), + expected_field="level_rules[].value", + expected_check="required", + ), + Scenario( + id="segment::level_rules[].between:linear_range_length", + scaffold={"level_rules": [{"value": 0, "between": [0.0, 1.0]}]}, + mutate=set_at_path("level_rules[].between", [0.5]), + expected_field="level_rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::level_rules[].between:linear_range_bounds", + scaffold={"level_rules": [{"value": 0, "between": [0.0, 1.0]}]}, + mutate=set_at_path("level_rules[].between", [1.5, 2.0]), + expected_field="level_rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::level_rules[].between:linear_range_order", + scaffold={"level_rules": [{"value": 0, "between": [0.0, 1.0]}]}, + mutate=set_at_path("level_rules[].between", [0.8, 0.2]), + expected_field="level_rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::routes[].name:string_min_length", + scaffold={"routes": [{"name": "a"}]}, + mutate=set_at_path("routes[].name", ""), + expected_field="routes[].name", + expected_check="string_min_length", + ), + Scenario( + id="segment::routes[].name:stripped", + scaffold={"routes": [{"name": "a"}]}, + mutate=set_at_path("routes[].name", " has spaces "), + expected_field="routes[].name", + expected_check="stripped", + ), + Scenario( + id="segment::routes[].network:string_min_length", + scaffold={"routes": [{"network": "a"}]}, + mutate=set_at_path("routes[].network", ""), + expected_field="routes[].network", + expected_check="string_min_length", + ), + Scenario( + id="segment::routes[].network:stripped", + scaffold={"routes": [{"network": "a"}]}, + mutate=set_at_path("routes[].network", " has spaces "), + expected_field="routes[].network", + expected_check="stripped", + ), + Scenario( + id="segment::routes[].ref:string_min_length", + scaffold={"routes": [{"ref": "a"}]}, + mutate=set_at_path("routes[].ref", ""), + expected_field="routes[].ref", + expected_check="string_min_length", + ), + Scenario( + id="segment::routes[].ref:stripped", + scaffold={"routes": [{"ref": "a"}]}, + mutate=set_at_path("routes[].ref", " has spaces "), + expected_field="routes[].ref", + expected_check="stripped", + ), + Scenario( + id="segment::routes[].symbol:string_min_length", + scaffold={"routes": [{"symbol": "a"}]}, + mutate=set_at_path("routes[].symbol", ""), + expected_field="routes[].symbol", + expected_check="string_min_length", + ), + Scenario( + id="segment::routes[].symbol:stripped", + scaffold={"routes": [{"symbol": "a"}]}, + mutate=set_at_path("routes[].symbol", " has spaces "), + expected_field="routes[].symbol", + expected_check="stripped", + ), + Scenario( + id="segment::routes[].wikidata:wikidata_id", + scaffold={"routes": [{"wikidata": "Q42"}]}, + mutate=set_at_path("routes[].wikidata", "P999"), + expected_field="routes[].wikidata", + expected_check="wikidata_id", + ), + Scenario( + id="segment::routes[].between:linear_range_length", + scaffold={"routes": [{"between": [0.0, 1.0]}]}, + mutate=set_at_path("routes[].between", [0.5]), + expected_field="routes[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::routes[].between:linear_range_bounds", + scaffold={"routes": [{"between": [0.0, 1.0]}]}, + mutate=set_at_path("routes[].between", [1.5, 2.0]), + expected_field="routes[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::routes[].between:linear_range_order", + scaffold={"routes": [{"between": [0.0, 1.0]}]}, + mutate=set_at_path("routes[].between", [0.8, 0.2]), + expected_field="routes[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::subclass_rules[].value:required", + scaffold={"subclass_rules": [{"value": "link"}]}, + mutate=set_at_path("subclass_rules[].value", None), + expected_field="subclass_rules[].value", + expected_check="required", + ), + Scenario( + id="segment::subclass_rules[].value:enum", + scaffold={"subclass_rules": [{"value": "link"}]}, + mutate=set_at_path("subclass_rules[].value", "__INVALID__"), + expected_field="subclass_rules[].value", + expected_check="enum", + ), + Scenario( + id="segment::subclass_rules[].between:linear_range_length", + scaffold={"subclass_rules": [{"value": "link", "between": [0.0, 1.0]}]}, + mutate=set_at_path("subclass_rules[].between", [0.5]), + expected_field="subclass_rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::subclass_rules[].between:linear_range_bounds", + scaffold={"subclass_rules": [{"value": "link", "between": [0.0, 1.0]}]}, + mutate=set_at_path("subclass_rules[].between", [1.5, 2.0]), + expected_field="subclass_rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::subclass_rules[].between:linear_range_order", + scaffold={"subclass_rules": [{"value": "link", "between": [0.0, 1.0]}]}, + mutate=set_at_path("subclass_rules[].between", [0.8, 0.2]), + expected_field="subclass_rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::names.primary:required", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", None), + expected_field="names.primary", + expected_check="required", + ), + Scenario( + id="segment::names.primary:string_min_length", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", ""), + expected_field="names.primary", + expected_check="string_min_length", + ), + Scenario( + id="segment::names.primary:stripped", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", " has spaces "), + expected_field="names.primary", + expected_check="stripped", + ), + Scenario( + id="segment::names.common{key}:language_tag", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_key(row, "names.common", "123"), + expected_field="names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="segment::names.common{value}:stripped", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_value(row, "names.common", " has spaces "), + expected_field="names.common{value}", + expected_check="stripped", + ), + Scenario( + id="segment::names.rules[].value:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", None), + expected_field="names.rules[].value", + expected_check="required", + ), + Scenario( + id="segment::names.rules[].value:string_min_length", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", ""), + expected_field="names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="segment::names.rules[].value:stripped", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", " has spaces "), + expected_field="names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="segment::names.rules[].variant:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", None), + expected_field="names.rules[].variant", + expected_check="required", + ), + Scenario( + id="segment::names.rules[].variant:enum", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", "__INVALID__"), + expected_field="names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="segment::names.rules[].language:language_tag", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + }, + mutate=set_at_path("names.rules[].language", "123"), + expected_field="names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="segment::names.rules[].perspectives.mode:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", None), + expected_field="names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="segment::names.rules[].perspectives.mode:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", "__INVALID__"), + expected_field="names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="segment::names.rules[].perspectives.countries:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", None), + expected_field="names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="segment::names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", []), + expected_field="names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=lambda row: mutate_unique_items( + row, "names.rules[].perspectives.countries" + ), + expected_field="names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries[]", "99"), + expected_field="names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="segment::names.rules[].between:linear_range_length", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.5]), + expected_field="names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::names.rules[].between:linear_range_bounds", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [1.5, 2.0]), + expected_field="names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::names.rules[].between:linear_range_order", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.8, 0.2]), + expected_field="names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::names.rules[].side:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + }, + mutate=set_at_path("names.rules[].side", "__INVALID__"), + expected_field="names.rules[].side", + expected_check="enum", + ), + Scenario( + id="segment::class_0:required", + scaffold={}, + mutate=set_at_path("class", None), + expected_field="class_0", + expected_check="required", + ), + Scenario( + id="segment::class_0:enum", + scaffold={}, + mutate=set_at_path("class", "__INVALID__"), + expected_field="class_0", + expected_check="enum", + ), + Scenario( + id="segment::destinations[].from_connector_id:required", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].from_connector_id", None), + expected_field="destinations[].from_connector_id", + expected_check="required", + ), + Scenario( + id="segment::destinations[].from_connector_id:string_min_length", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].from_connector_id", ""), + expected_field="destinations[].from_connector_id", + expected_check="string_min_length", + ), + Scenario( + id="segment::destinations[].from_connector_id:no_whitespace", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].from_connector_id", "has whitespace"), + expected_field="destinations[].from_connector_id", + expected_check="no_whitespace", + ), + Scenario( + id="segment::destinations[].to_connector_id:required", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].to_connector_id", None), + expected_field="destinations[].to_connector_id", + expected_check="required", + ), + Scenario( + id="segment::destinations[].to_connector_id:string_min_length", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].to_connector_id", ""), + expected_field="destinations[].to_connector_id", + expected_check="string_min_length", + ), + Scenario( + id="segment::destinations[].to_connector_id:no_whitespace", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].to_connector_id", "has whitespace"), + expected_field="destinations[].to_connector_id", + expected_check="no_whitespace", + ), + Scenario( + id="segment::destinations[].to_segment_id:required", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].to_segment_id", None), + expected_field="destinations[].to_segment_id", + expected_check="required", + ), + Scenario( + id="segment::destinations[].to_segment_id:string_min_length", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].to_segment_id", ""), + expected_field="destinations[].to_segment_id", + expected_check="string_min_length", + ), + Scenario( + id="segment::destinations[].to_segment_id:no_whitespace", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].to_segment_id", "has whitespace"), + expected_field="destinations[].to_segment_id", + expected_check="no_whitespace", + ), + Scenario( + id="segment::destinations[].final_heading:required", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].final_heading", None), + expected_field="destinations[].final_heading", + expected_check="required", + ), + Scenario( + id="segment::destinations[].final_heading:enum", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].final_heading", "__INVALID__"), + expected_field="destinations[].final_heading", + expected_check="enum", + ), + Scenario( + id="segment::destinations[].labels_min_length:array_min_length", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].labels", []), + expected_field="destinations[].labels_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::destinations[].labels_unique:struct_unique", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=lambda row: mutate_unique_items(row, "destinations[].labels"), + expected_field="destinations[].labels_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::destinations[].labels[].value:required", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].labels[].value", None), + expected_field="destinations[].labels[].value", + expected_check="required", + ), + Scenario( + id="segment::destinations[].labels[].value:string_min_length", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].labels[].value", ""), + expected_field="destinations[].labels[].value", + expected_check="string_min_length", + ), + Scenario( + id="segment::destinations[].labels[].value:stripped", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].labels[].value", " has spaces "), + expected_field="destinations[].labels[].value", + expected_check="stripped", + ), + Scenario( + id="segment::destinations[].labels[].type:required", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].labels[].type", None), + expected_field="destinations[].labels[].type", + expected_check="required", + ), + Scenario( + id="segment::destinations[].labels[].type:enum", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=set_at_path("destinations[].labels[].type", "__INVALID__"), + expected_field="destinations[].labels[].type", + expected_check="enum", + ), + Scenario( + id="segment::destinations[].symbols_unique:struct_unique", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + "symbols": ["motorway"], + } + ] + }, + mutate=lambda row: mutate_unique_items(row, "destinations[].symbols"), + expected_field="destinations[].symbols_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::destinations[].symbols[]:enum", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + "symbols": ["motorway"], + } + ] + }, + mutate=set_at_path("destinations[].symbols[]", "__INVALID__"), + expected_field="destinations[].symbols[]", + expected_check="enum", + ), + Scenario( + id="segment::destinations[].when.heading:required", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + "when": {"heading": "forward"}, + } + ] + }, + mutate=set_at_path("destinations[].when.heading", None), + expected_field="destinations[].when.heading", + expected_check="required", + ), + Scenario( + id="segment::destinations[].when.heading:enum", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + "when": {"heading": "forward"}, + } + ] + }, + mutate=set_at_path("destinations[].when.heading", "__INVALID__"), + expected_field="destinations[].when.heading", + expected_check="enum", + ), + Scenario( + id="segment::prohibited_transitions[].sequence:required", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + } + ] + }, + mutate=set_at_path("prohibited_transitions[].sequence", None), + expected_field="prohibited_transitions[].sequence", + expected_check="required", + ), + Scenario( + id="segment::prohibited_transitions[].sequence_min_length:array_min_length", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + } + ] + }, + mutate=set_at_path("prohibited_transitions[].sequence", []), + expected_field="prohibited_transitions[].sequence_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::prohibited_transitions[].sequence_unique:struct_unique", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + } + ] + }, + mutate=lambda row: mutate_unique_items( + row, "prohibited_transitions[].sequence" + ), + expected_field="prohibited_transitions[].sequence_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::prohibited_transitions[].sequence[].connector_id:required", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + } + ] + }, + mutate=set_at_path("prohibited_transitions[].sequence[].connector_id", None), + expected_field="prohibited_transitions[].sequence[].connector_id", + expected_check="required", + ), + Scenario( + id="segment::prohibited_transitions[].sequence[].connector_id:string_min_length", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + } + ] + }, + mutate=set_at_path("prohibited_transitions[].sequence[].connector_id", ""), + expected_field="prohibited_transitions[].sequence[].connector_id", + expected_check="string_min_length", + ), + Scenario( + id="segment::prohibited_transitions[].sequence[].connector_id:no_whitespace", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + } + ] + }, + mutate=set_at_path( + "prohibited_transitions[].sequence[].connector_id", "has whitespace" + ), + expected_field="prohibited_transitions[].sequence[].connector_id", + expected_check="no_whitespace", + ), + Scenario( + id="segment::prohibited_transitions[].sequence[].segment_id:required", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + } + ] + }, + mutate=set_at_path("prohibited_transitions[].sequence[].segment_id", None), + expected_field="prohibited_transitions[].sequence[].segment_id", + expected_check="required", + ), + Scenario( + id="segment::prohibited_transitions[].sequence[].segment_id:string_min_length", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + } + ] + }, + mutate=set_at_path("prohibited_transitions[].sequence[].segment_id", ""), + expected_field="prohibited_transitions[].sequence[].segment_id", + expected_check="string_min_length", + ), + Scenario( + id="segment::prohibited_transitions[].sequence[].segment_id:no_whitespace", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + } + ] + }, + mutate=set_at_path( + "prohibited_transitions[].sequence[].segment_id", "has whitespace" + ), + expected_field="prohibited_transitions[].sequence[].segment_id", + expected_check="no_whitespace", + ), + Scenario( + id="segment::prohibited_transitions[].final_heading:required", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + } + ] + }, + mutate=set_at_path("prohibited_transitions[].final_heading", None), + expected_field="prohibited_transitions[].final_heading", + expected_check="required", + ), + Scenario( + id="segment::prohibited_transitions[].final_heading:enum", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + } + ] + }, + mutate=set_at_path("prohibited_transitions[].final_heading", "__INVALID__"), + expected_field="prohibited_transitions[].final_heading", + expected_check="enum", + ), + Scenario( + id="segment::prohibited_transitions[].between:linear_range_length", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "between": [0.0, 1.0], + } + ] + }, + mutate=set_at_path("prohibited_transitions[].between", [0.5]), + expected_field="prohibited_transitions[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::prohibited_transitions[].between:linear_range_bounds", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "between": [0.0, 1.0], + } + ] + }, + mutate=set_at_path("prohibited_transitions[].between", [1.5, 2.0]), + expected_field="prohibited_transitions[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::prohibited_transitions[].between:linear_range_order", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "between": [0.0, 1.0], + } + ] + }, + mutate=set_at_path("prohibited_transitions[].between", [0.8, 0.2]), + expected_field="prohibited_transitions[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::prohibited_transitions[].when.heading:enum", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": {"heading": "forward"}, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.heading", "__INVALID__"), + expected_field="prohibited_transitions[].when.heading", + expected_check="enum", + ), + Scenario( + id="segment::prohibited_transitions[].when.mode_min_length:array_min_length", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.mode", []), + expected_field="prohibited_transitions[].when.mode_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::prohibited_transitions[].when.mode_unique:struct_unique", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items( + row, "prohibited_transitions[].when.mode" + ), + expected_field="prohibited_transitions[].when.mode_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::prohibited_transitions[].when.mode[]:enum", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.mode[]", "__INVALID__"), + expected_field="prohibited_transitions[].when.mode[]", + expected_check="enum", + ), + Scenario( + id="segment::prohibited_transitions[].when.using_min_length:array_min_length", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.using", []), + expected_field="prohibited_transitions[].when.using_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::prohibited_transitions[].when.using_unique:struct_unique", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items( + row, "prohibited_transitions[].when.using" + ), + expected_field="prohibited_transitions[].when.using_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::prohibited_transitions[].when.using[]:enum", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.using[]", "__INVALID__"), + expected_field="prohibited_transitions[].when.using[]", + expected_check="enum", + ), + Scenario( + id="segment::prohibited_transitions[].when.recognized_min_length:array_min_length", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.recognized", []), + expected_field="prohibited_transitions[].when.recognized_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::prohibited_transitions[].when.recognized_unique:struct_unique", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items( + row, "prohibited_transitions[].when.recognized" + ), + expected_field="prohibited_transitions[].when.recognized_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::prohibited_transitions[].when.recognized[]:enum", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.recognized[]", "__INVALID__"), + expected_field="prohibited_transitions[].when.recognized[]", + expected_check="enum", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle_min_length:array_min_length", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.vehicle", []), + expected_field="prohibited_transitions[].when.vehicle_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle_unique:struct_unique", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_unique_items( + row, "prohibited_transitions[].when.vehicle" + ), + expected_field="prohibited_transitions[].when.vehicle_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].dimension:required", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.vehicle[].dimension", None), + expected_field="prohibited_transitions[].when.vehicle[].dimension", + expected_check="required", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].dimension:enum", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path( + "prohibited_transitions[].when.vehicle[].dimension", "__INVALID__" + ), + expected_field="prohibited_transitions[].when.vehicle[].dimension", + expected_check="enum", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].comparison:required", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.vehicle[].comparison", None), + expected_field="prohibited_transitions[].when.vehicle[].comparison", + expected_check="required", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].comparison:enum", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path( + "prohibited_transitions[].when.vehicle[].comparison", "__INVALID__" + ), + expected_field="prohibited_transitions[].when.vehicle[].comparison", + expected_check="enum", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].value_0:required", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 0, + } + ], + }, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.vehicle[].value", None), + expected_field="prohibited_transitions[].when.vehicle[].value_0", + expected_check="required", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].value_1:required", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.vehicle[].value", None), + expected_field="prohibited_transitions[].when.vehicle[].value_1", + expected_check="required", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].value:bounds", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.vehicle[].value", -1.0), + expected_field="prohibited_transitions[].when.vehicle[].value", + expected_check="bounds", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].unit_0:required", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.vehicle[].unit", None), + expected_field="prohibited_transitions[].when.vehicle[].unit_0", + expected_check="required", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].unit_0:enum", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path( + "prohibited_transitions[].when.vehicle[].unit", "__INVALID__" + ), + expected_field="prohibited_transitions[].when.vehicle[].unit_0", + expected_check="enum", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].unit_1:required", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "weight", + "comparison": "greater_than", + "value": 0.0, + "unit": "oz", + } + ], + }, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.vehicle[].unit", None), + expected_field="prohibited_transitions[].when.vehicle[].unit_1", + expected_check="required", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].unit_1:enum", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "weight", + "comparison": "greater_than", + "value": 0.0, + "unit": "oz", + } + ], + }, + } + ] + }, + mutate=set_at_path( + "prohibited_transitions[].when.vehicle[].unit", "__INVALID__" + ), + expected_field="prohibited_transitions[].when.vehicle[].unit_1", + expected_check="enum", + ), + Scenario( + id="segment::road_flags_min_length:array_min_length", + scaffold={"road_flags": [{"values": ["is_bridge"]}]}, + mutate=set_at_path("road_flags", []), + expected_field="road_flags_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::road_flags_unique:struct_unique", + scaffold={"road_flags": [{"values": ["is_bridge"]}]}, + mutate=lambda row: mutate_unique_items(row, "road_flags"), + expected_field="road_flags_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::road_flags[].values:required", + scaffold={"road_flags": [{"values": ["is_bridge"]}]}, + mutate=set_at_path("road_flags[].values", None), + expected_field="road_flags[].values", + expected_check="required", + ), + Scenario( + id="segment::road_flags[].values_min_length:array_min_length", + scaffold={"road_flags": [{"values": ["is_bridge"]}]}, + mutate=set_at_path("road_flags[].values", []), + expected_field="road_flags[].values_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::road_flags[].values_unique:struct_unique", + scaffold={"road_flags": [{"values": ["is_bridge"]}]}, + mutate=lambda row: mutate_unique_items(row, "road_flags[].values"), + expected_field="road_flags[].values_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::road_flags[].values[]:enum", + scaffold={"road_flags": [{"values": ["is_bridge"]}]}, + mutate=set_at_path("road_flags[].values[]", "__INVALID__"), + expected_field="road_flags[].values[]", + expected_check="enum", + ), + Scenario( + id="segment::road_flags[].between:linear_range_length", + scaffold={"road_flags": [{"values": ["is_bridge"], "between": [0.0, 1.0]}]}, + mutate=set_at_path("road_flags[].between", [0.5]), + expected_field="road_flags[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::road_flags[].between:linear_range_bounds", + scaffold={"road_flags": [{"values": ["is_bridge"], "between": [0.0, 1.0]}]}, + mutate=set_at_path("road_flags[].between", [1.5, 2.0]), + expected_field="road_flags[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::road_flags[].between:linear_range_order", + scaffold={"road_flags": [{"values": ["is_bridge"], "between": [0.0, 1.0]}]}, + mutate=set_at_path("road_flags[].between", [0.8, 0.2]), + expected_field="road_flags[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::road_surface_min_length:array_min_length", + scaffold={"road_surface": [{"value": "unknown"}]}, + mutate=set_at_path("road_surface", []), + expected_field="road_surface_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::road_surface_unique:struct_unique", + scaffold={"road_surface": [{"value": "unknown"}]}, + mutate=lambda row: mutate_unique_items(row, "road_surface"), + expected_field="road_surface_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::road_surface[].value:required", + scaffold={"road_surface": [{"value": "unknown"}]}, + mutate=set_at_path("road_surface[].value", None), + expected_field="road_surface[].value", + expected_check="required", + ), + Scenario( + id="segment::road_surface[].value:enum", + scaffold={"road_surface": [{"value": "unknown"}]}, + mutate=set_at_path("road_surface[].value", "__INVALID__"), + expected_field="road_surface[].value", + expected_check="enum", + ), + Scenario( + id="segment::road_surface[].between:linear_range_length", + scaffold={"road_surface": [{"value": "unknown", "between": [0.0, 1.0]}]}, + mutate=set_at_path("road_surface[].between", [0.5]), + expected_field="road_surface[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::road_surface[].between:linear_range_bounds", + scaffold={"road_surface": [{"value": "unknown", "between": [0.0, 1.0]}]}, + mutate=set_at_path("road_surface[].between", [1.5, 2.0]), + expected_field="road_surface[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::road_surface[].between:linear_range_order", + scaffold={"road_surface": [{"value": "unknown", "between": [0.0, 1.0]}]}, + mutate=set_at_path("road_surface[].between", [0.8, 0.2]), + expected_field="road_surface[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::speed_limits_min_length:array_min_length", + scaffold={"speed_limits": [{"max_speed": {"value": 1, "unit": "mph"}}]}, + mutate=set_at_path("speed_limits", []), + expected_field="speed_limits_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::speed_limits_unique:struct_unique", + scaffold={"speed_limits": [{"max_speed": {"value": 1, "unit": "mph"}}]}, + mutate=lambda row: mutate_unique_items(row, "speed_limits"), + expected_field="speed_limits_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::speed_limits[].max_speed.value:required", + scaffold={"speed_limits": [{"max_speed": {"value": 1, "unit": "mph"}}]}, + mutate=set_at_path("speed_limits[].max_speed.value", None), + expected_field="speed_limits[].max_speed.value", + expected_check="required", + ), + Scenario( + id="segment::speed_limits[].max_speed.value_0:bounds", + scaffold={"speed_limits": [{"max_speed": {"value": 1, "unit": "mph"}}]}, + mutate=set_at_path("speed_limits[].max_speed.value", 0), + expected_field="speed_limits[].max_speed.value_0", + expected_check="bounds", + ), + Scenario( + id="segment::speed_limits[].max_speed.value_1:bounds", + scaffold={"speed_limits": [{"max_speed": {"value": 1, "unit": "mph"}}]}, + mutate=set_at_path("speed_limits[].max_speed.value", 351), + expected_field="speed_limits[].max_speed.value_1", + expected_check="bounds", + ), + Scenario( + id="segment::speed_limits[].max_speed.unit:required", + scaffold={"speed_limits": [{"max_speed": {"value": 1, "unit": "mph"}}]}, + mutate=set_at_path("speed_limits[].max_speed.unit", None), + expected_field="speed_limits[].max_speed.unit", + expected_check="required", + ), + Scenario( + id="segment::speed_limits[].max_speed.unit:enum", + scaffold={"speed_limits": [{"max_speed": {"value": 1, "unit": "mph"}}]}, + mutate=set_at_path("speed_limits[].max_speed.unit", "__INVALID__"), + expected_field="speed_limits[].max_speed.unit", + expected_check="enum", + ), + Scenario( + id="segment::speed_limits[].min_speed.value:required", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "min_speed": {"value": 1, "unit": "mph"}, + } + ] + }, + mutate=set_at_path("speed_limits[].min_speed.value", None), + expected_field="speed_limits[].min_speed.value", + expected_check="required", + ), + Scenario( + id="segment::speed_limits[].min_speed.value_0:bounds", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "min_speed": {"value": 1, "unit": "mph"}, + } + ] + }, + mutate=set_at_path("speed_limits[].min_speed.value", 0), + expected_field="speed_limits[].min_speed.value_0", + expected_check="bounds", + ), + Scenario( + id="segment::speed_limits[].min_speed.value_1:bounds", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "min_speed": {"value": 1, "unit": "mph"}, + } + ] + }, + mutate=set_at_path("speed_limits[].min_speed.value", 351), + expected_field="speed_limits[].min_speed.value_1", + expected_check="bounds", + ), + Scenario( + id="segment::speed_limits[].min_speed.unit:required", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "min_speed": {"value": 1, "unit": "mph"}, + } + ] + }, + mutate=set_at_path("speed_limits[].min_speed.unit", None), + expected_field="speed_limits[].min_speed.unit", + expected_check="required", + ), + Scenario( + id="segment::speed_limits[].min_speed.unit:enum", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "min_speed": {"value": 1, "unit": "mph"}, + } + ] + }, + mutate=set_at_path("speed_limits[].min_speed.unit", "__INVALID__"), + expected_field="speed_limits[].min_speed.unit", + expected_check="enum", + ), + Scenario( + id="segment::speed_limits[].between:linear_range_length", + scaffold={ + "speed_limits": [ + {"max_speed": {"value": 1, "unit": "mph"}, "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("speed_limits[].between", [0.5]), + expected_field="speed_limits[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::speed_limits[].between:linear_range_bounds", + scaffold={ + "speed_limits": [ + {"max_speed": {"value": 1, "unit": "mph"}, "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("speed_limits[].between", [1.5, 2.0]), + expected_field="speed_limits[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::speed_limits[].between:linear_range_order", + scaffold={ + "speed_limits": [ + {"max_speed": {"value": 1, "unit": "mph"}, "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("speed_limits[].between", [0.8, 0.2]), + expected_field="speed_limits[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::speed_limits[].when.heading:enum", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": {"heading": "forward"}, + } + ] + }, + mutate=set_at_path("speed_limits[].when.heading", "__INVALID__"), + expected_field="speed_limits[].when.heading", + expected_check="enum", + ), + Scenario( + id="segment::speed_limits[].when.mode_min_length:array_min_length", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=set_at_path("speed_limits[].when.mode", []), + expected_field="speed_limits[].when.mode_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::speed_limits[].when.mode_unique:struct_unique", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items(row, "speed_limits[].when.mode"), + expected_field="speed_limits[].when.mode_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::speed_limits[].when.mode[]:enum", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=set_at_path("speed_limits[].when.mode[]", "__INVALID__"), + expected_field="speed_limits[].when.mode[]", + expected_check="enum", + ), + Scenario( + id="segment::speed_limits[].when.using_min_length:array_min_length", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=set_at_path("speed_limits[].when.using", []), + expected_field="speed_limits[].when.using_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::speed_limits[].when.using_unique:struct_unique", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items(row, "speed_limits[].when.using"), + expected_field="speed_limits[].when.using_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::speed_limits[].when.using[]:enum", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=set_at_path("speed_limits[].when.using[]", "__INVALID__"), + expected_field="speed_limits[].when.using[]", + expected_check="enum", + ), + Scenario( + id="segment::speed_limits[].when.recognized_min_length:array_min_length", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=set_at_path("speed_limits[].when.recognized", []), + expected_field="speed_limits[].when.recognized_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::speed_limits[].when.recognized_unique:struct_unique", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items(row, "speed_limits[].when.recognized"), + expected_field="speed_limits[].when.recognized_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::speed_limits[].when.recognized[]:enum", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=set_at_path("speed_limits[].when.recognized[]", "__INVALID__"), + expected_field="speed_limits[].when.recognized[]", + expected_check="enum", + ), + Scenario( + id="segment::speed_limits[].when.vehicle_min_length:array_min_length", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle", []), + expected_field="speed_limits[].when.vehicle_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::speed_limits[].when.vehicle_unique:struct_unique", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_unique_items(row, "speed_limits[].when.vehicle"), + expected_field="speed_limits[].when.vehicle_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::speed_limits[].when.vehicle[].dimension:required", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].dimension", None), + expected_field="speed_limits[].when.vehicle[].dimension", + expected_check="required", + ), + Scenario( + id="segment::speed_limits[].when.vehicle[].dimension:enum", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].dimension", "__INVALID__"), + expected_field="speed_limits[].when.vehicle[].dimension", + expected_check="enum", + ), + Scenario( + id="segment::speed_limits[].when.vehicle[].comparison:required", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].comparison", None), + expected_field="speed_limits[].when.vehicle[].comparison", + expected_check="required", + ), + Scenario( + id="segment::speed_limits[].when.vehicle[].comparison:enum", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].comparison", "__INVALID__"), + expected_field="speed_limits[].when.vehicle[].comparison", + expected_check="enum", + ), + Scenario( + id="segment::speed_limits[].when.vehicle[].value_0:required", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 0, + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].value", None), + expected_field="speed_limits[].when.vehicle[].value_0", + expected_check="required", + ), + Scenario( + id="segment::speed_limits[].when.vehicle[].value_1:required", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].value", None), + expected_field="speed_limits[].when.vehicle[].value_1", + expected_check="required", + ), + Scenario( + id="segment::speed_limits[].when.vehicle[].value:bounds", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].value", -1.0), + expected_field="speed_limits[].when.vehicle[].value", + expected_check="bounds", + ), + Scenario( + id="segment::speed_limits[].when.vehicle[].unit_0:required", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].unit", None), + expected_field="speed_limits[].when.vehicle[].unit_0", + expected_check="required", + ), + Scenario( + id="segment::speed_limits[].when.vehicle[].unit_0:enum", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].unit", "__INVALID__"), + expected_field="speed_limits[].when.vehicle[].unit_0", + expected_check="enum", + ), + Scenario( + id="segment::speed_limits[].when.vehicle[].unit_1:required", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "weight", + "comparison": "greater_than", + "value": 0.0, + "unit": "oz", + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].unit", None), + expected_field="speed_limits[].when.vehicle[].unit_1", + expected_check="required", + ), + Scenario( + id="segment::speed_limits[].when.vehicle[].unit_1:enum", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "weight", + "comparison": "greater_than", + "value": 0.0, + "unit": "oz", + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].unit", "__INVALID__"), + expected_field="speed_limits[].when.vehicle[].unit_1", + expected_check="enum", + ), + Scenario( + id="segment::subclass:enum", + scaffold={"subclass": "link"}, + mutate=set_at_path("subclass", "__INVALID__"), + expected_field="subclass", + expected_check="enum", + ), + Scenario( + id="segment::width_rules_min_length:array_min_length", + scaffold={"width_rules": [{"value": 1.0}]}, + mutate=set_at_path("width_rules", []), + expected_field="width_rules_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::width_rules_unique:struct_unique", + scaffold={"width_rules": [{"value": 1.0}]}, + mutate=lambda row: mutate_unique_items(row, "width_rules"), + expected_field="width_rules_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::width_rules[].value:required", + scaffold={"width_rules": [{"value": 1.0}]}, + mutate=set_at_path("width_rules[].value", None), + expected_field="width_rules[].value", + expected_check="required", + ), + Scenario( + id="segment::width_rules[].value:bounds", + scaffold={"width_rules": [{"value": 1.0}]}, + mutate=set_at_path("width_rules[].value", 0.0), + expected_field="width_rules[].value", + expected_check="bounds", + ), + Scenario( + id="segment::width_rules[].between:linear_range_length", + scaffold={"width_rules": [{"value": 1.0, "between": [0.0, 1.0]}]}, + mutate=set_at_path("width_rules[].between", [0.5]), + expected_field="width_rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::width_rules[].between:linear_range_bounds", + scaffold={"width_rules": [{"value": 1.0, "between": [0.0, 1.0]}]}, + mutate=set_at_path("width_rules[].between", [1.5, 2.0]), + expected_field="width_rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::width_rules[].between:linear_range_order", + scaffold={"width_rules": [{"value": 1.0, "between": [0.0, 1.0]}]}, + mutate=set_at_path("width_rules[].between", [0.8, 0.2]), + expected_field="width_rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::model:forbid_if:0", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_forbid_if( + row, + ["unit"], + "dimension", + "axle_count", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:require_if:1", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "height", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_required_0", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:2", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "length", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_required_1", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:3", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "weight", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_required_2", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:4", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "width", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_required_3", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_any_of:5", + scaffold={ + "access_restrictions": [ + {"access_type": "allowed", "when": {"heading": "forward"}} + ] + }, + mutate=lambda row: mutate_require_any_of( + row, + ["heading", "during", "mode", "using", "recognized", "vehicle"], + array_path="access_restrictions", + struct_path="when", + ), + expected_field="access_restrictions[].when", + expected_check="require_any_of", + ), + Scenario( + id="segment::model:require_any_of:6", + scaffold={ + "destinations": [ + { + "from_connector_id": "a", + "to_connector_id": "a", + "to_segment_id": "a", + "final_heading": "forward", + "labels": [{"value": "a", "type": "street"}], + } + ] + }, + mutate=lambda row: mutate_require_any_of( + row, ["labels", "symbols"], array_path="destinations" + ), + expected_field="destinations[]", + expected_check="require_any_of", + ), + Scenario( + id="segment::model:forbid_if:7", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_forbid_if( + row, + ["unit"], + "dimension", + "axle_count", + array_path="prohibited_transitions", + inner_array_path="when.vehicle", + ), + expected_field="prohibited_transitions[].when.vehicle[].unit_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:require_if:8", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "height", + array_path="prohibited_transitions", + inner_array_path="when.vehicle", + ), + expected_field="prohibited_transitions[].when.vehicle[].unit_required_0", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:9", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "length", + array_path="prohibited_transitions", + inner_array_path="when.vehicle", + ), + expected_field="prohibited_transitions[].when.vehicle[].unit_required_1", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:10", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "weight", + array_path="prohibited_transitions", + inner_array_path="when.vehicle", + ), + expected_field="prohibited_transitions[].when.vehicle[].unit_required_2", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:11", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "width", + array_path="prohibited_transitions", + inner_array_path="when.vehicle", + ), + expected_field="prohibited_transitions[].when.vehicle[].unit_required_3", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_any_of:12", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": {"heading": "forward"}, + } + ] + }, + mutate=lambda row: mutate_require_any_of( + row, + ["heading", "during", "mode", "using", "recognized", "vehicle"], + array_path="prohibited_transitions", + struct_path="when", + ), + expected_field="prohibited_transitions[].when", + expected_check="require_any_of", + ), + Scenario( + id="segment::model:forbid_if:13", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_forbid_if( + row, + ["unit"], + "dimension", + "axle_count", + array_path="speed_limits", + inner_array_path="when.vehicle", + ), + expected_field="speed_limits[].when.vehicle[].unit_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:require_if:14", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "height", + array_path="speed_limits", + inner_array_path="when.vehicle", + ), + expected_field="speed_limits[].when.vehicle[].unit_required_0", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:15", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "length", + array_path="speed_limits", + inner_array_path="when.vehicle", + ), + expected_field="speed_limits[].when.vehicle[].unit_required_1", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:16", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "weight", + array_path="speed_limits", + inner_array_path="when.vehicle", + ), + expected_field="speed_limits[].when.vehicle[].unit_required_2", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:17", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "width", + array_path="speed_limits", + inner_array_path="when.vehicle", + ), + expected_field="speed_limits[].when.vehicle[].unit_required_3", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_any_of:18", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": {"heading": "forward"}, + } + ] + }, + mutate=lambda row: mutate_require_any_of( + row, + ["heading", "during", "mode", "using", "recognized", "vehicle"], + array_path="speed_limits", + struct_path="when", + ), + expected_field="speed_limits[].when", + expected_check="require_any_of", + ), + Scenario( + id="segment::model:require_any_of:19", + scaffold={"speed_limits": [{"max_speed": {"value": 1, "unit": "mph"}}]}, + mutate=lambda row: mutate_require_any_of( + row, ["max_speed.value", "min_speed.value"], array_path="speed_limits" + ), + expected_field="speed_limits[]", + expected_check="require_any_of", + ), + Scenario( + id="segment::model:forbid_if:20", + scaffold={}, + mutate=lambda row: mutate_forbid_if(row, ["class"], "subtype", "water"), + expected_field="class_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:require_if:21", + scaffold={}, + mutate=lambda row: mutate_require_if(row, ["class"], "subtype", "rail"), + expected_field="class_required_0", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:22", + scaffold={}, + mutate=lambda row: mutate_require_if(row, ["class"], "subtype", "road"), + expected_field="class_required_1", + expected_check="require_if", + ), + Scenario( + id="segment::model:forbid_if:23", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["destinations"], + "subtype", + "road", + negate=True, + fill_values={"destinations": [{}]}, + ), + expected_field="destinations_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:24", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["prohibited_transitions"], + "subtype", + "road", + negate=True, + fill_values={"prohibited_transitions": [{}]}, + ), + expected_field="prohibited_transitions_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:25", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["road_flags"], + "subtype", + "road", + negate=True, + fill_values={"road_flags": [{}]}, + ), + expected_field="road_flags_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:26", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["road_surface"], + "subtype", + "road", + negate=True, + fill_values={"road_surface": [{}]}, + ), + expected_field="road_surface_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:27", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["speed_limits"], + "subtype", + "road", + negate=True, + fill_values={"speed_limits": [{}]}, + ), + expected_field="speed_limits_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:28", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, ["subclass"], "subtype", "road", negate=True + ), + expected_field="subclass_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:29", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["width_rules"], + "subtype", + "road", + negate=True, + fill_values={"width_rules": [{}]}, + ), + expected_field="width_rules_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:30", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["rail_flags"], + "subtype", + "rail", + negate=True, + fill_values={"rail_flags": [{}]}, + ), + expected_field="rail_flags_forbidden", + expected_check="forbid_if", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return segment_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + SEGMENT_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="segment", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + SEGMENT_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="segment", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("segment::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("segment::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_water.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_water.py new file mode 100644 index 000000000..632f3757b --- /dev/null +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_water.py @@ -0,0 +1,1735 @@ +# Auto-generated — do not edit. + +"""Generated conformance tests for segment.""" + +from __future__ import annotations + +import pytest +from overture.schema.pyspark.expressions.generated.overture.schema.transportation.segment import ( + SEGMENT_SCHEMA, + segment_checks, +) +from pyspark.sql import SparkSession + +from ....._support.harness import ( + ValidationResults, + run_validation_pipeline, +) +from ....._support.helpers import set_at_path +from ....._support.mutations import ( + mutate_forbid_if, + mutate_map_key, + mutate_map_value, + mutate_require_any_of, + mutate_require_if, + mutate_unique_items, +) +from ....._support.scenarios import Scenario + +BASE_ROW_SPARSE: dict = { + "id": "1f4d65c9-e092-52c4-b002-7c11ce69a554", + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "transportation", + "type": "segment", + "version": 0, + "subtype": "water", +} + + +BASE_ROW_POPULATED: dict = { + "names": { + "primary": "a", + "common": {"en": "clean"}, + "rules": [ + { + "value": "a", + "variant": "common", + "language": "en", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + "between": [0.0, 1.0], + "side": "left", + } + ], + }, + "id": "1f4d65c9-e092-52c4-b002-7c11ce69a554", + "bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}, + "geometry": "LINESTRING (0 0, 1 1)", + "theme": "transportation", + "type": "segment", + "version": 0, + "sources": [ + { + "property": "/valid/pointer", + "dataset": "", + "license": "clean", + "record_id": "", + "update_time": "2024-01-01T00:00:00Z", + "confidence": 0.0, + "provider": "a", + "resource": "a", + "version": "a", + "between": [0.0, 1.0], + } + ], + "subtype": "water", + "access_restrictions": [ + { + "access_type": "allowed", + "between": [0.0, 1.0], + "when": { + "heading": "forward", + "during": "", + "mode": ["vehicle"], + "using": ["as_customer"], + "recognized": ["as_permitted"], + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ], + "connectors": [{"connector_id": "a", "at": 0.0}, {"connector_id": "a1", "at": 0.0}], + "level_rules": [{"value": 0, "between": [0.0, 1.0]}], + "routes": [ + { + "name": "a", + "network": "a", + "ref": "a", + "symbol": "a", + "wikidata": "Q42", + "between": [0.0, 1.0], + } + ], + "subclass_rules": [{"value": "link", "between": [0.0, 1.0]}], +} + + +SCENARIOS: list[Scenario] = [ + Scenario( + id="segment::id:required", + scaffold={}, + mutate=set_at_path("id", None), + expected_field="id", + expected_check="required", + ), + Scenario( + id="segment::id:string_min_length", + scaffold={}, + mutate=set_at_path("id", ""), + expected_field="id", + expected_check="string_min_length", + ), + Scenario( + id="segment::id:no_whitespace", + scaffold={}, + mutate=set_at_path("id", "has whitespace"), + expected_field="id", + expected_check="no_whitespace", + ), + Scenario( + id="segment::bbox:bbox_completeness", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": None, "ymax": 1.0} + ), + expected_field="bbox", + expected_check="bbox_completeness", + ), + Scenario( + id="segment::bbox:bbox_lat_ordering", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": 10.0, "ymax": -10.0} + ), + expected_field="bbox", + expected_check="bbox_lat_ordering", + ), + Scenario( + id="segment::bbox:bbox_lat_range", + scaffold={"bbox": {"xmin": 0.0, "xmax": 1.0, "ymin": 0.0, "ymax": 1.0}}, + mutate=set_at_path( + "bbox", {"xmin": 0.0, "xmax": 1.0, "ymin": -100.0, "ymax": 100.0} + ), + expected_field="bbox", + expected_check="bbox_lat_range", + ), + Scenario( + id="segment::geometry:required", + scaffold={}, + mutate=set_at_path("geometry", None), + expected_field="geometry", + expected_check="required", + ), + Scenario( + id="segment::geometry:geometry_type", + scaffold={}, + mutate=set_at_path("geometry", "POINT (0 0)"), + expected_field="geometry", + expected_check="geometry_type", + ), + Scenario( + id="segment::theme:required", + scaffold={}, + mutate=set_at_path("theme", None), + expected_field="theme", + expected_check="required", + ), + Scenario( + id="segment::theme:enum", + scaffold={}, + mutate=set_at_path("theme", "__INVALID__"), + expected_field="theme", + expected_check="enum", + ), + Scenario( + id="segment::type:required", + scaffold={}, + mutate=set_at_path("type", None), + expected_field="type", + expected_check="required", + ), + Scenario( + id="segment::type:enum", + scaffold={}, + mutate=set_at_path("type", "__INVALID__"), + expected_field="type", + expected_check="enum", + ), + Scenario( + id="segment::version:required", + scaffold={}, + mutate=set_at_path("version", None), + expected_field="version", + expected_check="required", + ), + Scenario( + id="segment::version:bounds", + scaffold={}, + mutate=set_at_path("version", -1), + expected_field="version", + expected_check="bounds", + ), + Scenario( + id="segment::sources_min_length:array_min_length", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources", []), + expected_field="sources_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::sources_unique:struct_unique", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=lambda row: mutate_unique_items(row, "sources"), + expected_field="sources_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::sources[].property:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", None), + expected_field="sources[].property", + expected_check="required", + ), + Scenario( + id="segment::sources[].property:json_pointer", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].property", "no-slash"), + expected_field="sources[].property", + expected_check="json_pointer", + ), + Scenario( + id="segment::sources[].dataset:required", + scaffold={"sources": [{"property": "/valid/pointer", "dataset": ""}]}, + mutate=set_at_path("sources[].dataset", None), + expected_field="sources[].dataset", + expected_check="required", + ), + Scenario( + id="segment::sources[].license:stripped", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "license": "clean"} + ] + }, + mutate=set_at_path("sources[].license", " has spaces "), + expected_field="sources[].license", + expected_check="stripped", + ), + Scenario( + id="segment::sources[].confidence_0:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", -1.0), + expected_field="sources[].confidence_0", + expected_check="bounds", + ), + Scenario( + id="segment::sources[].confidence_1:bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "confidence": 0.0} + ] + }, + mutate=set_at_path("sources[].confidence", 2.0), + expected_field="sources[].confidence_1", + expected_check="bounds", + ), + Scenario( + id="segment::sources[].provider:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", ""), + expected_field="sources[].provider", + expected_check="string_min_length", + ), + Scenario( + id="segment::sources[].provider:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "provider": "a"}] + }, + mutate=set_at_path("sources[].provider", "HAS SPACES"), + expected_field="sources[].provider", + expected_check="snake_case", + ), + Scenario( + id="segment::sources[].resource:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", ""), + expected_field="sources[].resource", + expected_check="string_min_length", + ), + Scenario( + id="segment::sources[].resource:snake_case", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "resource": "a"}] + }, + mutate=set_at_path("sources[].resource", "HAS SPACES"), + expected_field="sources[].resource", + expected_check="snake_case", + ), + Scenario( + id="segment::sources[].version:string_min_length", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", ""), + expected_field="sources[].version", + expected_check="string_min_length", + ), + Scenario( + id="segment::sources[].version:no_whitespace", + scaffold={ + "sources": [{"property": "/valid/pointer", "dataset": "", "version": "a"}] + }, + mutate=set_at_path("sources[].version", "has whitespace"), + expected_field="sources[].version", + expected_check="no_whitespace", + ), + Scenario( + id="segment::sources[].between:linear_range_length", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.5]), + expected_field="sources[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::sources[].between:linear_range_bounds", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [1.5, 2.0]), + expected_field="sources[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::sources[].between:linear_range_order", + scaffold={ + "sources": [ + {"property": "/valid/pointer", "dataset": "", "between": [0.0, 1.0]} + ] + }, + mutate=set_at_path("sources[].between", [0.8, 0.2]), + expected_field="sources[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::subtype:required", + scaffold={}, + mutate=set_at_path("subtype", None), + expected_field="subtype", + expected_check="required", + ), + Scenario( + id="segment::subtype:enum", + scaffold={}, + mutate=set_at_path("subtype", "__INVALID__"), + expected_field="subtype", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions_min_length:array_min_length", + scaffold={"access_restrictions": [{"access_type": "allowed"}]}, + mutate=set_at_path("access_restrictions", []), + expected_field="access_restrictions_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions_unique:struct_unique", + scaffold={"access_restrictions": [{"access_type": "allowed"}]}, + mutate=lambda row: mutate_unique_items(row, "access_restrictions"), + expected_field="access_restrictions_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].access_type:required", + scaffold={"access_restrictions": [{"access_type": "allowed"}]}, + mutate=set_at_path("access_restrictions[].access_type", None), + expected_field="access_restrictions[].access_type", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].access_type:enum", + scaffold={"access_restrictions": [{"access_type": "allowed"}]}, + mutate=set_at_path("access_restrictions[].access_type", "__INVALID__"), + expected_field="access_restrictions[].access_type", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].between:linear_range_length", + scaffold={ + "access_restrictions": [{"access_type": "allowed", "between": [0.0, 1.0]}] + }, + mutate=set_at_path("access_restrictions[].between", [0.5]), + expected_field="access_restrictions[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::access_restrictions[].between:linear_range_bounds", + scaffold={ + "access_restrictions": [{"access_type": "allowed", "between": [0.0, 1.0]}] + }, + mutate=set_at_path("access_restrictions[].between", [1.5, 2.0]), + expected_field="access_restrictions[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::access_restrictions[].between:linear_range_order", + scaffold={ + "access_restrictions": [{"access_type": "allowed", "between": [0.0, 1.0]}] + }, + mutate=set_at_path("access_restrictions[].between", [0.8, 0.2]), + expected_field="access_restrictions[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::access_restrictions[].when.heading:enum", + scaffold={ + "access_restrictions": [ + {"access_type": "allowed", "when": {"heading": "forward"}} + ] + }, + mutate=set_at_path("access_restrictions[].when.heading", "__INVALID__"), + expected_field="access_restrictions[].when.heading", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.mode_min_length:array_min_length", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.mode", []), + expected_field="access_restrictions[].when.mode_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions[].when.mode_unique:struct_unique", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items(row, "access_restrictions[].when.mode"), + expected_field="access_restrictions[].when.mode_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].when.mode[]:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "mode": ["vehicle"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.mode[]", "__INVALID__"), + expected_field="access_restrictions[].when.mode[]", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.using_min_length:array_min_length", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.using", []), + expected_field="access_restrictions[].when.using_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions[].when.using_unique:struct_unique", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items(row, "access_restrictions[].when.using"), + expected_field="access_restrictions[].when.using_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].when.using[]:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "using": ["as_customer"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.using[]", "__INVALID__"), + expected_field="access_restrictions[].when.using[]", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.recognized_min_length:array_min_length", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.recognized", []), + expected_field="access_restrictions[].when.recognized_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions[].when.recognized_unique:struct_unique", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=lambda row: mutate_unique_items( + row, "access_restrictions[].when.recognized" + ), + expected_field="access_restrictions[].when.recognized_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].when.recognized[]:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": {"heading": "forward", "recognized": ["as_permitted"]}, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.recognized[]", "__INVALID__"), + expected_field="access_restrictions[].when.recognized[]", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle_min_length:array_min_length", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle", []), + expected_field="access_restrictions[].when.vehicle_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle_unique:struct_unique", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_unique_items( + row, "access_restrictions[].when.vehicle" + ), + expected_field="access_restrictions[].when.vehicle_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].dimension:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].dimension", None), + expected_field="access_restrictions[].when.vehicle[].dimension", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].dimension:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path( + "access_restrictions[].when.vehicle[].dimension", "__INVALID__" + ), + expected_field="access_restrictions[].when.vehicle[].dimension", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].comparison:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].comparison", None), + expected_field="access_restrictions[].when.vehicle[].comparison", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].comparison:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path( + "access_restrictions[].when.vehicle[].comparison", "__INVALID__" + ), + expected_field="access_restrictions[].when.vehicle[].comparison", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value_0:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 0, + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", None), + expected_field="access_restrictions[].when.vehicle[].value_0", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value_1:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", None), + expected_field="access_restrictions[].when.vehicle[].value_1", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value:bounds", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", -1.0), + expected_field="access_restrictions[].when.vehicle[].value", + expected_check="bounds", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].unit_0:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].unit", None), + expected_field="access_restrictions[].when.vehicle[].unit_0", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].unit_0:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].unit", "__INVALID__"), + expected_field="access_restrictions[].when.vehicle[].unit_0", + expected_check="enum", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].unit_1:required", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "weight", + "comparison": "greater_than", + "value": 0.0, + "unit": "oz", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].unit", None), + expected_field="access_restrictions[].when.vehicle[].unit_1", + expected_check="required", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].unit_1:enum", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "weight", + "comparison": "greater_than", + "value": 0.0, + "unit": "oz", + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].unit", "__INVALID__"), + expected_field="access_restrictions[].when.vehicle[].unit_1", + expected_check="enum", + ), + Scenario( + id="segment::connectors_min_length:array_min_length", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=set_at_path("connectors", []), + expected_field="connectors_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::connectors_unique:struct_unique", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=lambda row: mutate_unique_items(row, "connectors"), + expected_field="connectors_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::connectors[].connector_id:required", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=set_at_path("connectors[].connector_id", None), + expected_field="connectors[].connector_id", + expected_check="required", + ), + Scenario( + id="segment::connectors[].connector_id:string_min_length", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=set_at_path("connectors[].connector_id", ""), + expected_field="connectors[].connector_id", + expected_check="string_min_length", + ), + Scenario( + id="segment::connectors[].connector_id:no_whitespace", + scaffold={"connectors": [{"connector_id": "a"}, {"connector_id": "a1"}]}, + mutate=set_at_path("connectors[].connector_id", "has whitespace"), + expected_field="connectors[].connector_id", + expected_check="no_whitespace", + ), + Scenario( + id="segment::connectors[].at_0:bounds", + scaffold={ + "connectors": [{"connector_id": "a", "at": 0.0}, {"connector_id": "a1"}] + }, + mutate=set_at_path("connectors[].at", -1.0), + expected_field="connectors[].at_0", + expected_check="bounds", + ), + Scenario( + id="segment::connectors[].at_1:bounds", + scaffold={ + "connectors": [{"connector_id": "a", "at": 0.0}, {"connector_id": "a1"}] + }, + mutate=set_at_path("connectors[].at", 2.0), + expected_field="connectors[].at_1", + expected_check="bounds", + ), + Scenario( + id="segment::level_rules[].value:required", + scaffold={"level_rules": [{"value": 0}]}, + mutate=set_at_path("level_rules[].value", None), + expected_field="level_rules[].value", + expected_check="required", + ), + Scenario( + id="segment::level_rules[].between:linear_range_length", + scaffold={"level_rules": [{"value": 0, "between": [0.0, 1.0]}]}, + mutate=set_at_path("level_rules[].between", [0.5]), + expected_field="level_rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::level_rules[].between:linear_range_bounds", + scaffold={"level_rules": [{"value": 0, "between": [0.0, 1.0]}]}, + mutate=set_at_path("level_rules[].between", [1.5, 2.0]), + expected_field="level_rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::level_rules[].between:linear_range_order", + scaffold={"level_rules": [{"value": 0, "between": [0.0, 1.0]}]}, + mutate=set_at_path("level_rules[].between", [0.8, 0.2]), + expected_field="level_rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::routes[].name:string_min_length", + scaffold={"routes": [{"name": "a"}]}, + mutate=set_at_path("routes[].name", ""), + expected_field="routes[].name", + expected_check="string_min_length", + ), + Scenario( + id="segment::routes[].name:stripped", + scaffold={"routes": [{"name": "a"}]}, + mutate=set_at_path("routes[].name", " has spaces "), + expected_field="routes[].name", + expected_check="stripped", + ), + Scenario( + id="segment::routes[].network:string_min_length", + scaffold={"routes": [{"network": "a"}]}, + mutate=set_at_path("routes[].network", ""), + expected_field="routes[].network", + expected_check="string_min_length", + ), + Scenario( + id="segment::routes[].network:stripped", + scaffold={"routes": [{"network": "a"}]}, + mutate=set_at_path("routes[].network", " has spaces "), + expected_field="routes[].network", + expected_check="stripped", + ), + Scenario( + id="segment::routes[].ref:string_min_length", + scaffold={"routes": [{"ref": "a"}]}, + mutate=set_at_path("routes[].ref", ""), + expected_field="routes[].ref", + expected_check="string_min_length", + ), + Scenario( + id="segment::routes[].ref:stripped", + scaffold={"routes": [{"ref": "a"}]}, + mutate=set_at_path("routes[].ref", " has spaces "), + expected_field="routes[].ref", + expected_check="stripped", + ), + Scenario( + id="segment::routes[].symbol:string_min_length", + scaffold={"routes": [{"symbol": "a"}]}, + mutate=set_at_path("routes[].symbol", ""), + expected_field="routes[].symbol", + expected_check="string_min_length", + ), + Scenario( + id="segment::routes[].symbol:stripped", + scaffold={"routes": [{"symbol": "a"}]}, + mutate=set_at_path("routes[].symbol", " has spaces "), + expected_field="routes[].symbol", + expected_check="stripped", + ), + Scenario( + id="segment::routes[].wikidata:wikidata_id", + scaffold={"routes": [{"wikidata": "Q42"}]}, + mutate=set_at_path("routes[].wikidata", "P999"), + expected_field="routes[].wikidata", + expected_check="wikidata_id", + ), + Scenario( + id="segment::routes[].between:linear_range_length", + scaffold={"routes": [{"between": [0.0, 1.0]}]}, + mutate=set_at_path("routes[].between", [0.5]), + expected_field="routes[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::routes[].between:linear_range_bounds", + scaffold={"routes": [{"between": [0.0, 1.0]}]}, + mutate=set_at_path("routes[].between", [1.5, 2.0]), + expected_field="routes[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::routes[].between:linear_range_order", + scaffold={"routes": [{"between": [0.0, 1.0]}]}, + mutate=set_at_path("routes[].between", [0.8, 0.2]), + expected_field="routes[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::subclass_rules[].value:required", + scaffold={"subclass_rules": [{"value": "link"}]}, + mutate=set_at_path("subclass_rules[].value", None), + expected_field="subclass_rules[].value", + expected_check="required", + ), + Scenario( + id="segment::subclass_rules[].value:enum", + scaffold={"subclass_rules": [{"value": "link"}]}, + mutate=set_at_path("subclass_rules[].value", "__INVALID__"), + expected_field="subclass_rules[].value", + expected_check="enum", + ), + Scenario( + id="segment::subclass_rules[].between:linear_range_length", + scaffold={"subclass_rules": [{"value": "link", "between": [0.0, 1.0]}]}, + mutate=set_at_path("subclass_rules[].between", [0.5]), + expected_field="subclass_rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::subclass_rules[].between:linear_range_bounds", + scaffold={"subclass_rules": [{"value": "link", "between": [0.0, 1.0]}]}, + mutate=set_at_path("subclass_rules[].between", [1.5, 2.0]), + expected_field="subclass_rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::subclass_rules[].between:linear_range_order", + scaffold={"subclass_rules": [{"value": "link", "between": [0.0, 1.0]}]}, + mutate=set_at_path("subclass_rules[].between", [0.8, 0.2]), + expected_field="subclass_rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::names.primary:required", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", None), + expected_field="names.primary", + expected_check="required", + ), + Scenario( + id="segment::names.primary:string_min_length", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", ""), + expected_field="names.primary", + expected_check="string_min_length", + ), + Scenario( + id="segment::names.primary:stripped", + scaffold={"names": {"primary": "a"}}, + mutate=set_at_path("names.primary", " has spaces "), + expected_field="names.primary", + expected_check="stripped", + ), + Scenario( + id="segment::names.common{key}:language_tag", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_key(row, "names.common", "123"), + expected_field="names.common{key}", + expected_check="language_tag", + ), + Scenario( + id="segment::names.common{value}:stripped", + scaffold={"names": {"primary": "a", "common": {"en": "clean"}}}, + mutate=lambda row: mutate_map_value(row, "names.common", " has spaces "), + expected_field="names.common{value}", + expected_check="stripped", + ), + Scenario( + id="segment::names.rules[].value:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", None), + expected_field="names.rules[].value", + expected_check="required", + ), + Scenario( + id="segment::names.rules[].value:string_min_length", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", ""), + expected_field="names.rules[].value", + expected_check="string_min_length", + ), + Scenario( + id="segment::names.rules[].value:stripped", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].value", " has spaces "), + expected_field="names.rules[].value", + expected_check="stripped", + ), + Scenario( + id="segment::names.rules[].variant:required", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", None), + expected_field="names.rules[].variant", + expected_check="required", + ), + Scenario( + id="segment::names.rules[].variant:enum", + scaffold={ + "names": {"primary": "a", "rules": [{"value": "a", "variant": "common"}]} + }, + mutate=set_at_path("names.rules[].variant", "__INVALID__"), + expected_field="names.rules[].variant", + expected_check="enum", + ), + Scenario( + id="segment::names.rules[].language:language_tag", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "language": "en"}], + } + }, + mutate=set_at_path("names.rules[].language", "123"), + expected_field="names.rules[].language", + expected_check="language_tag", + ), + Scenario( + id="segment::names.rules[].perspectives.mode:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", None), + expected_field="names.rules[].perspectives.mode", + expected_check="required", + ), + Scenario( + id="segment::names.rules[].perspectives.mode:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.mode", "__INVALID__"), + expected_field="names.rules[].perspectives.mode", + expected_check="enum", + ), + Scenario( + id="segment::names.rules[].perspectives.countries:required", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", None), + expected_field="names.rules[].perspectives.countries", + expected_check="required", + ), + Scenario( + id="segment::names.rules[].perspectives.countries_min_length:array_min_length", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries", []), + expected_field="names.rules[].perspectives.countries_min_length", + expected_check="array_min_length", + ), + Scenario( + id="segment::names.rules[].perspectives.countries_unique:struct_unique", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=lambda row: mutate_unique_items( + row, "names.rules[].perspectives.countries" + ), + expected_field="names.rules[].perspectives.countries_unique", + expected_check="struct_unique", + ), + Scenario( + id="segment::names.rules[].perspectives.countries[]:country_code_alpha2", + scaffold={ + "names": { + "primary": "a", + "rules": [ + { + "value": "a", + "variant": "common", + "perspectives": {"mode": "accepted_by", "countries": ["US"]}, + } + ], + } + }, + mutate=set_at_path("names.rules[].perspectives.countries[]", "99"), + expected_field="names.rules[].perspectives.countries[]", + expected_check="country_code_alpha2", + ), + Scenario( + id="segment::names.rules[].between:linear_range_length", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.5]), + expected_field="names.rules[].between", + expected_check="linear_range_length", + ), + Scenario( + id="segment::names.rules[].between:linear_range_bounds", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [1.5, 2.0]), + expected_field="names.rules[].between", + expected_check="linear_range_bounds", + ), + Scenario( + id="segment::names.rules[].between:linear_range_order", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "between": [0.0, 1.0]}], + } + }, + mutate=set_at_path("names.rules[].between", [0.8, 0.2]), + expected_field="names.rules[].between", + expected_check="linear_range_order", + ), + Scenario( + id="segment::names.rules[].side:enum", + scaffold={ + "names": { + "primary": "a", + "rules": [{"value": "a", "variant": "common", "side": "left"}], + } + }, + mutate=set_at_path("names.rules[].side", "__INVALID__"), + expected_field="names.rules[].side", + expected_check="enum", + ), + Scenario( + id="segment::model:forbid_if:0", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_forbid_if( + row, + ["unit"], + "dimension", + "axle_count", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:require_if:1", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "height", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_required_0", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:2", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "length", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_required_1", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:3", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "weight", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_required_2", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:4", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "height", + "comparison": "greater_than", + "value": 0.0, + "unit": "in", + } + ], + }, + } + ] + }, + mutate=lambda row: mutate_require_if( + row, + ["unit"], + "dimension", + "width", + array_path="access_restrictions", + inner_array_path="when.vehicle", + ), + expected_field="access_restrictions[].when.vehicle[].unit_required_3", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_any_of:5", + scaffold={ + "access_restrictions": [ + {"access_type": "allowed", "when": {"heading": "forward"}} + ] + }, + mutate=lambda row: mutate_require_any_of( + row, + ["heading", "during", "mode", "using", "recognized", "vehicle"], + array_path="access_restrictions", + struct_path="when", + ), + expected_field="access_restrictions[].when", + expected_check="require_any_of", + ), + Scenario( + id="segment::model:forbid_if:6", + scaffold={}, + mutate=lambda row: mutate_forbid_if(row, ["class"], "subtype", "water"), + expected_field="class_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:require_if:7", + scaffold={}, + mutate=lambda row: mutate_require_if(row, ["class"], "subtype", "rail"), + expected_field="class_required_0", + expected_check="require_if", + ), + Scenario( + id="segment::model:require_if:8", + scaffold={}, + mutate=lambda row: mutate_require_if(row, ["class"], "subtype", "road"), + expected_field="class_required_1", + expected_check="require_if", + ), + Scenario( + id="segment::model:forbid_if:9", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["destinations"], + "subtype", + "road", + negate=True, + fill_values={"destinations": [{}]}, + ), + expected_field="destinations_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:10", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["prohibited_transitions"], + "subtype", + "road", + negate=True, + fill_values={"prohibited_transitions": [{}]}, + ), + expected_field="prohibited_transitions_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:11", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["road_flags"], + "subtype", + "road", + negate=True, + fill_values={"road_flags": [{}]}, + ), + expected_field="road_flags_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:12", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["road_surface"], + "subtype", + "road", + negate=True, + fill_values={"road_surface": [{}]}, + ), + expected_field="road_surface_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:13", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["speed_limits"], + "subtype", + "road", + negate=True, + fill_values={"speed_limits": [{}]}, + ), + expected_field="speed_limits_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:14", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, ["subclass"], "subtype", "road", negate=True + ), + expected_field="subclass_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:15", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["width_rules"], + "subtype", + "road", + negate=True, + fill_values={"width_rules": [{}]}, + ), + expected_field="width_rules_forbidden", + expected_check="forbid_if", + ), + Scenario( + id="segment::model:forbid_if:16", + scaffold={}, + mutate=lambda row: mutate_forbid_if( + row, + ["rail_flags"], + "subtype", + "rail", + negate=True, + fill_values={"rail_flags": [{}]}, + ), + expected_field="rail_flags_forbidden", + expected_check="forbid_if", + ), +] + + +@pytest.fixture(scope="module") +def checks() -> list: + return segment_checks() + + +@pytest.fixture(scope="module") +def sparse_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + SEGMENT_SCHEMA, + checks, + BASE_ROW_SPARSE, + SCENARIOS, + model_name="segment", + ) + + +@pytest.fixture(scope="module") +def populated_results(spark: SparkSession, checks: list) -> ValidationResults: + return run_validation_pipeline( + spark, + SEGMENT_SCHEMA, + checks, + BASE_ROW_POPULATED, + SCENARIOS, + model_name="segment", + ) + + +def test_baseline_sparse(sparse_results: ValidationResults) -> None: + """Sparse base row passes every check the codegen produced. + + Catches drift between base_row synthesis, schema_builder, and + check_builder -- if any of those produce output inconsistent with + the others (e.g. a check that rejects values the synthesizer emits + for required-only fields), the baseline fails here before any + scenario runs. + """ + baseline = sparse_results.violations.get("segment::baseline", set()) + assert baseline == set(), f"Sparse baseline has violations: {baseline}" + + +def test_baseline_populated(populated_results: ValidationResults) -> None: + """Fully-populated base row passes every check the codegen produced. + + Mirrors `test_baseline_sparse` but with all optional fields + filled, exercising codegen paths that only fire when a value is + present. + """ + baseline = populated_results.violations.get("segment::baseline", set()) + assert baseline == set(), f"Populated baseline has violations: {baseline}" + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_sparse( + scenario: Scenario, + sparse_results: ValidationResults, +) -> None: + _assert_scenario(scenario, sparse_results) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.id) +def test_scenario_populated( + scenario: Scenario, + populated_results: ValidationResults, +) -> None: + _assert_scenario(scenario, populated_results) + + +def _assert_scenario( + scenario: Scenario, + validation_results: ValidationResults, +) -> None: + expected = (scenario.expected_field, scenario.expected_check) + if scenario.id in validation_results.skipped: + # An unbuildable scenario exercises nothing; fail loud rather than skip + # (a skip reads as a pass and hides codegen/scaffold gaps). + pytest.fail( + f"unbuildable scenario {scenario.id!r}: " + f"{validation_results.skipped[scenario.id]}" + ) + valid_violations = validation_results.violations.get(f"{scenario.id}::valid", set()) + assert expected not in valid_violations + invalid_violations = validation_results.violations.get( + f"{scenario.id}::invalid", set() + ) + assert expected in invalid_violations diff --git a/packages/overture-schema-pyspark/tests/test_check.py b/packages/overture-schema-pyspark/tests/test_check.py new file mode 100644 index 000000000..0993411c2 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/test_check.py @@ -0,0 +1,20 @@ +"""Tests for Check dataclass and CheckShape enum.""" + +import dataclasses + +import pytest +from overture.schema.pyspark.check import Check, CheckShape +from pyspark.sql import SparkSession +from pyspark.sql import functions as F + + +def test_check_is_frozen(spark: SparkSession) -> None: + check = Check( + field="subtype", + name="required", + expr=F.lit("error"), + shape=CheckShape.SCALAR, + read_columns=frozenset({"subtype"}), + ) + with pytest.raises(dataclasses.FrozenInstanceError): + check.field = "other" # type: ignore[misc] diff --git a/packages/overture-schema-pyspark/tests/test_cli.py b/packages/overture-schema-pyspark/tests/test_cli.py new file mode 100644 index 000000000..04e573efa --- /dev/null +++ b/packages/overture-schema-pyspark/tests/test_cli.py @@ -0,0 +1,632 @@ +"""Tests for CLI entry points.""" + +from collections.abc import Iterator +from pathlib import Path + +import pytest +from click.testing import CliRunner +from overture.schema.pyspark._registry import REGISTRY +from overture.schema.pyspark.check import Check, CheckShape +from overture.schema.pyspark.cli import ( + ReadSpec, + _spark_config, + absent_column, + read_feature, + resolve_read, + validate_cli, +) +from pyspark.errors import AnalysisException +from pyspark.sql import Row, SparkSession +from pyspark.sql import functions as F +from pyspark.sql.types import StringType, StructField, StructType + +from ._support.registry import register_model + +_TEST_TYPE = "_test_cli" + +# Shared schema for all test registrations that need the four base columns. +_BASE_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("value", StringType(), True), + ] +) + +# Tests that branch on registered geometry types require the runtime registry +# to be populated (i.e. generated expression modules present). +_requires_generated = pytest.mark.skipif( + not REGISTRY, reason="requires generated expression modules" +) + + +class TestSparkConfig: + """Tests for S3A auto-configuration.""" + + @_requires_generated + def test_large_geometry_disables_vectorized_reader(self) -> None: + config = _spark_config( + "samples/segment.parquet", (), "overture.schema.transportation:Segment" + ) + assert config["spark.sql.parquet.enableVectorizedReader"] == "false" + + @_requires_generated + def test_point_geometry_keeps_vectorized_reader(self) -> None: + config = _spark_config( + "samples/place.parquet", (), "overture.schema.places:Place" + ) + assert "spark.sql.parquet.enableVectorizedReader" not in config + + def test_unspecified_geometry_disables_vectorized_reader(self) -> None: + # _TEST_TYPE registers no geometry_types -- safe default disables the reader + config = _spark_config("samples/test.parquet", (), _TEST_TYPE) + assert config["spark.sql.parquet.enableVectorizedReader"] == "false" + + def test_s3a_path_applies_defaults(self) -> None: + config = _spark_config("s3a://bucket/path", (), _TEST_TYPE) + assert "org.apache.hadoop:hadoop-aws" in config["spark.jars.packages"] + assert "S3AFileSystem" in config["spark.hadoop.fs.s3a.impl"] + assert ( + "AnonymousAWSCredentialsProvider" + in config["spark.hadoop.fs.s3a.aws.credentials.provider"] + ) + + def test_user_conf_overrides_s3a_defaults(self) -> None: + config = _spark_config( + "s3a://bucket/path", + ( + "spark.hadoop.fs.s3a.aws.credentials.provider=" + "software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider", + ), + _TEST_TYPE, + ) + assert ( + "ProfileCredentialsProvider" + in config["spark.hadoop.fs.s3a.aws.credentials.provider"] + ) + + def test_user_conf_merges_with_s3a_defaults(self) -> None: + config = _spark_config( + "s3a://bucket/path", ("spark.master=local[4]",), _TEST_TYPE + ) + assert config["spark.master"] == "local[4]" + assert "spark.jars.packages" in config + + def test_local_path_passes_user_conf(self) -> None: + config = _spark_config( + "samples/test.parquet", ("spark.master=local[4]",), _TEST_TYPE + ) + assert config["spark.master"] == "local[4]" + assert config["spark.sql.parquet.enableVectorizedReader"] == "false" + + +def _test_checks() -> list[Check]: + """Minimal checks for CLI testing: value must be 'good'.""" + return [ + Check( + field="value", + name="enum", + expr=F.when(F.col("value") != "good", F.lit("not good")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"value"}), + ), + ] + + +@pytest.fixture(autouse=True) +def _register_test_checks() -> Iterator[None]: + with register_model(_TEST_TYPE, _BASE_SCHEMA, _test_checks): + yield + + +def test_validate_missing_args() -> None: + runner = CliRunner() + result = runner.invoke(validate_cli, []) + assert result.exit_code != 0 + + +def test_validate_unknown_type() -> None: + runner = CliRunner() + result = runner.invoke(validate_cli, ["nonexistent", "/dev/null"]) + assert result.exit_code != 0 + assert "nonexistent" in result.output + + +def test_validate_clean_data(spark: SparkSession, tmp_path: Path) -> None: + """Valid data exits 0, no output file written.""" + input_path = str(tmp_path / "input.parquet") + output_path = str(tmp_path / "output.parquet") + + spark.createDataFrame( + [Row(id="r1", theme="test", type="test_cli", value="good")] + ).write.parquet(input_path) + + runner = CliRunner() + result = runner.invoke(validate_cli, [_TEST_TYPE, input_path, "-o", output_path]) + assert result.exit_code == 0, result.output + assert "0 / 1 rows with errors" in result.output + assert not Path(output_path).exists() + + +def test_validate_error_count(spark: SparkSession, tmp_path: Path) -> None: + """Rows with errors are counted in summary.""" + input_path = str(tmp_path / "input.parquet") + output_path = str(tmp_path / "output.parquet") + + spark.createDataFrame( + [Row(id="r1", theme="test", type="test_cli", value="bad")] + ).write.parquet(input_path) + + runner = CliRunner() + result = runner.invoke(validate_cli, [_TEST_TYPE, input_path, "-o", output_path]) + assert result.exit_code != 0 + assert "1 / 1 rows with errors" in result.output + + +def test_validate_shows_error_rows(spark: SparkSession, tmp_path: Path) -> None: + """Error rows are displayed with violation columns.""" + input_path = str(tmp_path / "input.parquet") + output_path = str(tmp_path / "output.parquet") + + spark.createDataFrame( + [Row(id="row1", theme="test", type="test_cli", value="bad")] + ).write.parquet(input_path) + + runner = CliRunner() + result = runner.invoke(validate_cli, [_TEST_TYPE, input_path, "-o", output_path]) + assert result.exit_code != 0 + assert "row1" in result.output + assert "value" in result.output + + +def test_validate_head_zero(spark: SparkSession, tmp_path: Path) -> None: + """--head 0 suppresses the error row table.""" + input_path = str(tmp_path / "input.parquet") + output_path = str(tmp_path / "output.parquet") + + spark.createDataFrame( + [Row(id="row1", theme="test", type="test_cli", value="bad")] + ).write.parquet(input_path) + + runner = CliRunner() + result = runner.invoke( + validate_cli, [_TEST_TYPE, input_path, "-o", output_path, "--head", "0"] + ) + assert result.exit_code != 0 + assert "1 / 1 rows with errors" in result.output + assert "row1" not in result.output + + +def test_validate_schema_mismatch_exits(spark: SparkSession, tmp_path: Path) -> None: + """Schema mismatch prints diff and exits before validation.""" + input_path = str(tmp_path / "input.parquet") + output_path = str(tmp_path / "output.parquet") + + # Write data with wrong schema (IntegerType where StringType expected) + spark.createDataFrame( + [Row(id="r1", value=42)], schema="id string, value int" + ).write.parquet(input_path) + + runner = CliRunner() + result = runner.invoke(validate_cli, [_TEST_TYPE, input_path, "-o", output_path]) + assert result.exit_code != 0 + assert "Schema mismatch" in result.output + assert "value" in result.output + + +def test_validate_skip_schema_check(spark: SparkSession, tmp_path: Path) -> None: + """--skip-schema-check warns on mismatches but continues validation.""" + input_path = str(tmp_path / "input.parquet") + + # Extra column causes a mismatch but doesn't break check evaluation + spark.createDataFrame( + [Row(id="r1", theme="test", type="test_cli", value="good", extra="x")] + ).write.parquet(input_path) + + runner = CliRunner() + result = runner.invoke( + validate_cli, [_TEST_TYPE, input_path, "--skip-schema-check"] + ) + assert "Schema mismatch" in result.output + assert "rows with errors" in result.output + + +def test_validate_skip_columns(spark: SparkSession, tmp_path: Path) -> None: + """--skip-columns skips checks for absent columns.""" + input_path = str(tmp_path / "input.parquet") + + # Data missing 'value' column — declare it absent via --skip-columns + spark.createDataFrame([Row(id="r1", theme="test", type="test_cli")]).write.parquet( + input_path + ) + + runner = CliRunner() + result = runner.invoke( + validate_cli, + [_TEST_TYPE, input_path, "--skip-columns", "value", "--skip-schema-check"], + ) + assert result.exit_code == 0, result.output + assert "0 / 1 rows with errors" in result.output + + +def test_validate_missing_column_suggests_skip_columns( + spark: SparkSession, tmp_path: Path +) -> None: + """A column absent from the data hints the --skip-columns flag.""" + input_path = str(tmp_path / "input.parquet") + + # Data missing the 'value' column the schema expects + spark.createDataFrame([Row(id="r1", theme="test", type="test_cli")]).write.parquet( + input_path + ) + + runner = CliRunner() + result = runner.invoke(validate_cli, [_TEST_TYPE, input_path]) + assert result.exit_code != 0 + assert "Schema mismatch" in result.output + assert "--skip-columns value" in result.output + + +def _unresolved(object_name: str, *, suggestion: bool = True) -> AnalysisException: + """An UNRESOLVED_COLUMN AnalysisException naming `object_name`.""" + suffix = "WITH_SUGGESTION" if suggestion else "WITHOUT_SUGGESTION" + return AnalysisException( + f"column {object_name} cannot be resolved", + errorClass=f"UNRESOLVED_COLUMN.{suffix}", + messageParameters={"objectName": object_name}, + ) + + +class TestAbsentColumn: + """Classification of AnalysisExceptions into absent-column vs. bug.""" + + def test_absent_top_level_column_is_named(self) -> None: + assert absent_column(_unresolved("`phantom`"), ["id", "value"]) == "phantom" + + def test_without_suggestion_is_also_named(self) -> None: + exc = _unresolved("`phantom`", suggestion=False) + assert absent_column(exc, ["id", "value"]) == "phantom" + + def test_dotted_reference_yields_top_level_column(self) -> None: + assert absent_column(_unresolved("`bbox`.`xmin`"), ["id"]) == "bbox" + + def test_present_column_is_not_named(self) -> None: + # An unresolved-column error naming a column that *is* present is not + # the missing-data case --skip-columns resolves; treat it as a bug. + assert absent_column(_unresolved("`value`"), ["id", "value"]) is None + + def test_non_unresolved_condition_is_a_bug(self) -> None: + exc = AnalysisException( + "cannot extract field from scalar", + errorClass="INVALID_EXTRACT_BASE_FIELD_TYPE", + messageParameters={"base": '"value"', "other": '"STRING"'}, + ) + assert absent_column(exc, ["id", "value"]) is None + + def test_missing_condition_is_a_bug(self) -> None: + assert absent_column(AnalysisException("opaque failure"), ["id"]) is None + + def test_missing_object_name_is_a_bug(self) -> None: + exc = AnalysisException( + "unresolved with no objectName", + errorClass="UNRESOLVED_COLUMN.WITHOUT_SUGGESTION", + messageParameters={}, + ) + assert absent_column(exc, ["id"]) is None + + +def test_validate_unresolvable_check_names_absent_column( + spark: SparkSession, tmp_path: Path +) -> None: + """An unresolved-column check names the absent column and hints --skip-columns.""" + unresolvable_type = "_test_cli_unresolvable" + # A check reading a column in neither the data nor the expected schema + # is invisible to absence detection (which only flags expected-but- + # missing columns), so it survives the read_columns drop, reaches + # evaluation, and raises UNRESOLVED_COLUMN -- the backstop the CLI + # must catch and convert into a column-named hint. + with register_model( + unresolvable_type, + _BASE_SCHEMA, + checks=lambda: [ + Check( + field="phantom", + name="present", + expr=F.when(F.col("phantom").isNull(), F.lit("missing phantom")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"phantom"}), + ) + ], + ): + input_path = str(tmp_path / "input.parquet") + spark.createDataFrame( + [Row(id="r1", theme="test", type="x", value="good")] + ).write.parquet(input_path) + runner = CliRunner() + result = runner.invoke(validate_cli, [unresolvable_type, input_path]) + assert result.exit_code != 0 + assert "phantom" in result.output + assert "--skip-columns phantom" in result.output + + +def test_validate_planning_bug_propagates(spark: SparkSession, tmp_path: Path) -> None: + """A non-column planning error surfaces as a bug, not a --skip-columns hint.""" + buggy_type = "_test_cli_planning_bug" + # `value` is a present string column, so the check survives the + # read_columns drop, but extracting a struct field from a string is a + # generator bug: it raises an AnalysisException that is *not* + # UNRESOLVED_COLUMN. `--skip-columns value` would not fix it, so the + # backstop must let it propagate rather than masking it. + with register_model( + buggy_type, + _BASE_SCHEMA, + checks=lambda: [ + Check( + field="value", + name="struct_field", + expr=F.when(F.col("value").getField("missing").isNull(), F.lit("bad")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"value"}), + ) + ], + ): + input_path = str(tmp_path / "input.parquet") + spark.createDataFrame( + [Row(id="r1", theme="test", type="x", value="good")] + ).write.parquet(input_path) + runner = CliRunner() + result = runner.invoke(validate_cli, [buggy_type, input_path]) + assert result.exit_code != 0 + assert isinstance(result.exception, AnalysisException) + assert "--skip-columns" not in result.output + + +def test_validate_ignore_extra_columns(spark: SparkSession, tmp_path: Path) -> None: + """--ignore-extra-columns suppresses 'expected missing' schema mismatches.""" + input_path = str(tmp_path / "input.parquet") + + spark.createDataFrame( + [Row(id="r1", theme="test", type="test_cli", value="good", extra="x")] + ).write.parquet(input_path) + + runner = CliRunner() + # Without the flag, schema mismatch exits + result = runner.invoke(validate_cli, [_TEST_TYPE, input_path]) + assert result.exit_code != 0 + assert "Schema mismatch" in result.output + + # With the flag, extra column is tolerated + result = runner.invoke( + validate_cli, [_TEST_TYPE, input_path, "--ignore-extra-columns", "extra"] + ) + assert result.exit_code == 0, result.output + assert "0 / 1 rows with errors" in result.output + + +def test_validate_suppress_field(spark: SparkSession, tmp_path: Path) -> None: + """--suppress FIELD removes all checks on that field.""" + input_path = str(tmp_path / "input.parquet") + + spark.createDataFrame( + [Row(id="r1", theme="test", type="test_cli", value="bad")] + ).write.parquet(input_path) + + runner = CliRunner() + result = runner.invoke( + validate_cli, [_TEST_TYPE, input_path, "--suppress", "value"] + ) + assert result.exit_code == 0, result.output + assert "0 / 1 rows with errors" in result.output + + +def test_validate_suppress_field_check(spark: SparkSession, tmp_path: Path) -> None: + """--suppress FIELD:CHECK removes a specific check.""" + input_path = str(tmp_path / "input.parquet") + + spark.createDataFrame( + [Row(id="r1", theme="test", type="test_cli", value="bad")] + ).write.parquet(input_path) + + runner = CliRunner() + result = runner.invoke( + validate_cli, [_TEST_TYPE, input_path, "--suppress", "value:enum"] + ) + assert result.exit_code == 0, result.output + assert "0 / 1 rows with errors" in result.output + + +def test_validate_output_contains_explained_violations( + spark: SparkSession, tmp_path: Path +) -> None: + """Output Parquet contains explain() violations with field/check/message.""" + input_path = str(tmp_path / "input.parquet") + output_path = str(tmp_path / "output.parquet") + + spark.createDataFrame( + [ + Row(id="r1", theme="test", type="test_cli", value="good"), + Row(id="r2", theme="test", type="test_cli", value="bad"), + ] + ).write.parquet(input_path) + + runner = CliRunner() + runner.invoke(validate_cli, [_TEST_TYPE, input_path, "-o", output_path]) + + result_df = spark.read.parquet(output_path) + assert {"field", "check", "message"} <= set(result_df.columns) + assert result_df.count() == 1 # one violation from r2 + + +_BATHYMETRY_PARTITIONS = {"theme": "base", "type": "bathymetry"} +_SEGMENT_PARTITIONS = {"theme": "transportation", "type": "segment"} + + +class TestResolveRead: + """Pure-function tests for path resolution logic.""" + + def test_release_root(self) -> None: + spec = resolve_read("/data/release/2026-02-18.0/", _BATHYMETRY_PARTITIONS) + assert spec == ReadSpec( + data_path="/data/release/2026-02-18.0/theme=base/type=bathymetry", + base_path="/data/release/2026-02-18.0", + ) + + def test_release_root_no_trailing_slash(self) -> None: + spec = resolve_read("/data/release/2026-02-18.0", _BATHYMETRY_PARTITIONS) + assert spec == ReadSpec( + data_path="/data/release/2026-02-18.0/theme=base/type=bathymetry", + base_path="/data/release/2026-02-18.0", + ) + + def test_leaf_partition(self) -> None: + spec = resolve_read( + "/data/release/2026-02-18.0/theme=base/type=bathymetry/", + _BATHYMETRY_PARTITIONS, + ) + assert spec == ReadSpec( + data_path="/data/release/2026-02-18.0/theme=base/type=bathymetry/", + base_path="/data/release/2026-02-18.0", + ) + + def test_theme_partition_appends_type_leaf(self) -> None: + # A theme-level path is missing the `type=` leaf; resolve_read must + # append it so a single feature's checks aren't run against every + # type sharing the theme directory. + spec = resolve_read( + "/data/release/2026-02-18.0/theme=base/", _BATHYMETRY_PARTITIONS + ) + assert spec == ReadSpec( + data_path="/data/release/2026-02-18.0/theme=base/type=bathymetry", + base_path="/data/release/2026-02-18.0", + ) + + def test_individual_file(self) -> None: + spec = resolve_read("/tmp/bathymetry.parquet", _BATHYMETRY_PARTITIONS) + assert spec == ReadSpec(data_path="/tmp/bathymetry.parquet") + + def test_individual_file_no_partitions(self) -> None: + spec = resolve_read("/tmp/data.parquet", None) + assert spec == ReadSpec(data_path="/tmp/data.parquet") + + def test_plain_directory_no_partitions(self) -> None: + spec = resolve_read("/tmp/data/", None) + assert spec == ReadSpec(data_path="/tmp/data/") + + def test_s3a_release_root(self) -> None: + spec = resolve_read("s3a://bucket/release/2026-02-18.0/", _SEGMENT_PARTITIONS) + assert spec == ReadSpec( + data_path="s3a://bucket/release/2026-02-18.0/theme=transportation/type=segment", + base_path="s3a://bucket/release/2026-02-18.0", + ) + + def test_s3a_leaf_partition(self) -> None: + spec = resolve_read( + "s3a://bucket/release/2026-02-18.0/theme=transportation/type=segment/", + _SEGMENT_PARTITIONS, + ) + assert spec == ReadSpec( + data_path="s3a://bucket/release/2026-02-18.0/theme=transportation/type=segment/", + base_path="s3a://bucket/release/2026-02-18.0", + ) + + +def _write_partitioned(spark: SparkSession, base_dir: Path, rows: list[Row]) -> None: + """Write test rows as Hive-partitioned Parquet under *base_dir*.""" + spark.createDataFrame(rows).write.partitionBy("theme", "type").parquet( + str(base_dir) + ) + + +class TestReadFeature: + """Integration tests: resolve_read + read_feature against local Parquet.""" + + def test_read_from_release_root(self, spark: SparkSession, tmp_path: Path) -> None: + base = tmp_path / "release" + _write_partitioned( + spark, + base, + [Row(id="r1", value="good", theme="test", type=_TEST_TYPE)], + ) + spec = resolve_read(str(base), {"theme": "test", "type": _TEST_TYPE}) + df = read_feature(spark, spec) + assert df.count() == 1 + assert set(df.columns) >= {"id", "theme", "type", "value"} + + def test_read_from_leaf_partition( + self, spark: SparkSession, tmp_path: Path + ) -> None: + base = tmp_path / "release" + _write_partitioned( + spark, + base, + [Row(id="r1", value="good", theme="test", type=_TEST_TYPE)], + ) + leaf = str(base / f"theme=test/type={_TEST_TYPE}") + spec = resolve_read(leaf, {"theme": "test", "type": _TEST_TYPE}) + df = read_feature(spark, spec) + assert df.count() == 1 + assert set(df.columns) >= {"id", "theme", "type", "value"} + + def test_read_from_individual_file( + self, spark: SparkSession, tmp_path: Path + ) -> None: + file_path = str(tmp_path / "data.parquet") + spark.createDataFrame( + [Row(id="r1", theme="test", type=_TEST_TYPE, value="good")] + ).write.parquet(file_path) + spec = resolve_read(file_path, {"theme": "test", "type": _TEST_TYPE}) + df = read_feature(spark, spec) + assert df.count() == 1 + assert set(df.columns) >= {"id", "theme", "type", "value"} + + def test_release_root_filters_to_type( + self, spark: SparkSession, tmp_path: Path + ) -> None: + """Only the target type's rows are returned from a multi-type release.""" + base = tmp_path / "release" + _write_partitioned( + spark, + base, + [ + Row(id="r1", value="good", theme="test", type=_TEST_TYPE), + Row(id="r2", value="good", theme="test", type="other"), + ], + ) + spec = resolve_read(str(base), {"theme": "test", "type": _TEST_TYPE}) + df = read_feature(spark, spec) + assert df.count() == 1 + assert df.collect()[0]["id"] == "r1" + + def test_theme_partition_filters_to_type( + self, spark: SparkSession, tmp_path: Path + ) -> None: + """A theme-level path returns only the target type, not its siblings.""" + base = tmp_path / "release" + _write_partitioned( + spark, + base, + [ + Row(id="r1", value="good", theme="test", type=_TEST_TYPE), + Row(id="r2", value="good", theme="test", type="other"), + ], + ) + theme_path = str(base / "theme=test") + spec = resolve_read(theme_path, {"theme": "test", "type": _TEST_TYPE}) + df = read_feature(spark, spec) + assert df.count() == 1 + assert df.collect()[0]["id"] == "r1" + + +def test_validate_from_partitioned_release(spark: SparkSession, tmp_path: Path) -> None: + """Full CLI round-trip reading from a Hive-partitioned release root.""" + base = tmp_path / "release" + _write_partitioned( + spark, + base, + [Row(id="r1", value="good", theme="test", type=_TEST_TYPE)], + ) + runner = CliRunner() + result = runner.invoke(validate_cli, [_TEST_TYPE, str(base)]) + assert result.exit_code == 0, result.output + assert "0 / 1 rows with errors" in result.output diff --git a/packages/overture-schema-pyspark/tests/test_harness.py b/packages/overture-schema-pyspark/tests/test_harness.py new file mode 100644 index 000000000..3fa07d628 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/test_harness.py @@ -0,0 +1,447 @@ +"""Tests for the conformance test harness.""" + +from __future__ import annotations + +import re + +import pytest +from overture.schema.pyspark.check import Check, CheckShape +from pyspark.sql import Row, SparkSession +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + +from ._support.harness import ( + assert_schema_covers_checks, + build_scenario_map, + build_scenario_rows, + coerce_to_schema, + index_violations, + sanitize_row, + scenario_uuid, +) +from ._support.helpers import PathTraversalError, set_at_path +from ._support.scenarios import Scenario + + +class TestScenarioUuid: + def test_deterministic(self) -> None: + """Same ID produces same UUID.""" + assert scenario_uuid("building::id:required") == scenario_uuid( + "building::id:required" + ) + + def test_different_ids_different_uuids(self) -> None: + assert scenario_uuid("a::b:c") != scenario_uuid("d::e:f") + + def test_valid_uuid_format(self) -> None: + uuid_str = scenario_uuid("test::x:y") + assert re.match( + r"^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$", + uuid_str, + ) + + +class TestBuildScenarioMap: + def test_scenarios_get_valid_and_invalid_entries(self) -> None: + scenarios = [ + Scenario( + id="f::x:required", + scaffold={}, + mutate=set_at_path("x", None), + expected_field="x", + expected_check="required", + ), + ] + scenario_map = build_scenario_map(scenarios, model_name="f") + assert scenario_uuid("f::x:required::valid") in scenario_map + assert ( + scenario_map[scenario_uuid("f::x:required::valid")] + == "f::x:required::valid" + ) + assert scenario_uuid("f::x:required::invalid") in scenario_map + assert ( + scenario_map[scenario_uuid("f::x:required::invalid")] + == "f::x:required::invalid" + ) + + def test_baseline_plus_two_entries_per_scenario(self) -> None: + scenarios = [ + Scenario( + id="f::x:check", + scaffold={}, + mutate=set_at_path("x", 0), + expected_field="x", + expected_check="check", + ), + ] + scenario_map = build_scenario_map(scenarios, model_name="f") + # baseline + (::valid, ::invalid) for the one scenario + assert len(scenario_map) == 3 + + def test_duplicate_id_values_raises(self) -> None: + scenarios = [ + Scenario( + id="f::x:required", + scaffold={}, + mutate=set_at_path("x", None), + expected_field="x", + expected_check="required", + ), + Scenario( + id="f::x:required", + scaffold={}, + mutate=set_at_path("x", None), + expected_field="x", + expected_check="required", + ), + ] + with pytest.raises(ValueError, match="Duplicate"): + build_scenario_map(scenarios, model_name="f") + + +class TestBuildScenarioRows: + def test_baseline_row_included(self) -> None: + base = {"id": "original-uuid", "theme": "buildings", "type": "building", "x": 1} + rows, scenario_map, skipped = build_scenario_rows( + base, [], model_name="building" + ) + assert len(rows) == 1 + assert rows[0]["theme"] == "buildings" + assert "_scenario_id" in rows[0] + + def test_path_traversal_error_skips(self) -> None: + """Mutation functions that raise PathTraversalError produce skips.""" + base = {"theme": "t", "type": "ty"} + + def bad_mutation(row: dict) -> dict: + raise PathTraversalError("cannot traverse") + + scenarios = [ + Scenario( + id="f::x:check", + scaffold={}, + mutate=bad_mutation, + expected_field="x", + expected_check="check", + ), + ] + rows, scenario_map, skipped = build_scenario_rows( + base, scenarios, model_name="f" + ) + assert len(rows) == 1 + assert "f::x:check" in skipped + + def test_scenario_creates_valid_and_invalid_rows(self) -> None: + """Each Scenario produces both a valid and an invalid row.""" + base = {"id": "orig", "theme": "t", "type": "ty", "x": 1} + scenarios = [ + Scenario( + id="f::x:required", + scaffold={}, + mutate=set_at_path("x", None), + expected_field="x", + expected_check="required", + ), + ] + rows, scenario_map, skipped = build_scenario_rows( + base, scenarios, model_name="f" + ) + # baseline + valid + invalid + assert len(rows) == 3 + assert rows[1]["x"] == 1 # valid row is a copy of base_row + assert rows[2]["x"] is None + assert rows[1]["_scenario_id"] == scenario_uuid("f::x:required::valid") + assert rows[2]["_scenario_id"] == scenario_uuid("f::x:required::invalid") + + def test_valid_row_uses_scaffold_merged_row(self) -> None: + """Valid row merges the scaffold so the target is present and exercised. + + The scaffold reaches a target the base row lacks; the valid row must + carry it (with no mutation) or the no-violation assertion is vacuous. + """ + base = {"id": "orig", "theme": "t", "type": "ty"} + scenarios = [ + Scenario( + id="f::items[].a:required", + scaffold={"items": [{"a": 0}]}, + mutate=set_at_path("items[].a", None), + expected_field="items[].a", + expected_check="required", + ), + ] + rows, scenario_map, skipped = build_scenario_rows( + base, scenarios, model_name="f" + ) + assert len(rows) == 3 + # Valid row carries the scaffold's target value (no mutation). + assert rows[1]["items"] == [{"a": 0}] + # Invalid row applies the mutation on top of the scaffold. + assert rows[2]["items"][0]["a"] is None + + def test_valid_scaffold_overrides_scaffold_for_valid_row(self) -> None: + """`valid_scaffold`, when set, builds the valid row instead of `scaffold`. + + The invalid row still uses `scaffold` -- only the valid row diverges, + e.g. to seed a literal alternative the mutation scaffold can't carry. + """ + base = {"id": "orig", "theme": "t", "type": "ty"} + scenarios = [ + Scenario( + id="f::kind:literal", + scaffold={"kind": "synthesized"}, + mutate=set_at_path("kind", None), + expected_field="kind", + expected_check="required", + valid_scaffold={"kind": "literal-alt"}, + ), + ] + rows, _scenario_map, _skipped = build_scenario_rows( + base, scenarios, model_name="f" + ) + valid_id = scenario_uuid("f::kind:literal::valid") + invalid_id = scenario_uuid("f::kind:literal::invalid") + valid_row = next(r for r in rows if r["_scenario_id"] == valid_id) + invalid_row = next(r for r in rows if r["_scenario_id"] == invalid_id) + assert valid_row["kind"] == "literal-alt" + assert invalid_row["kind"] is None + + def test_scaffold_merged_onto_invalid_row(self) -> None: + base_row = {"id": "x", "a": 1} + s = Scenario( + id="test::b:check", + scaffold={"b": 10}, + mutate=set_at_path("b", 0), + expected_field="b", + expected_check="check", + ) + rows, scenario_map, skipped = build_scenario_rows( + base_row, [s], model_name="test" + ) + invalid_id = scenario_uuid("test::b:check::invalid") + invalid_row = next(r for r in rows if r["_scenario_id"] == invalid_id) + # base field preserved, scaffold provides b, path overrides b + assert invalid_row["a"] == 1 + assert invalid_row["b"] == 0 + + def test_applies_scaffold_then_mutation(self) -> None: + base_row = {"id": "x", "a": 1} + s = Scenario( + id="test::model:check", + scaffold={"b": 10}, + mutate=lambda row: {**row, "a": None}, + expected_field="a", + expected_check="required", + ) + rows, scenario_map, skipped = build_scenario_rows( + base_row, [s], model_name="test" + ) + assert len(rows) == 3 + assert not skipped + invalid_id = scenario_uuid("test::model:check::invalid") + invalid_row = next(r for r in rows if r["_scenario_id"] == invalid_id) + # scaffold merged: b exists + assert invalid_row["b"] == 10 + # mutation applied: a is None + assert invalid_row["a"] is None + + +class TestCoerceToSchema: + """Ints land as floats in float columns; Spark would otherwise null them.""" + + def test_int_in_double_column_becomes_float(self) -> None: + schema = StructType([StructField("v", DoubleType(), True)]) + result = coerce_to_schema({"v": 0}, schema) + assert isinstance(result["v"], float) + assert result["v"] == 0.0 + + def test_bool_in_double_column_left_alone(self) -> None: + """`bool` is an `int` subclass but maps to BooleanType, not a float.""" + schema = StructType([StructField("v", DoubleType(), True)]) + assert coerce_to_schema({"v": True}, schema)["v"] is True + + def test_int_column_unchanged(self) -> None: + schema = StructType([StructField("v", IntegerType(), True)]) + assert coerce_to_schema({"v": 5}, schema)["v"] == 5 + + def test_nested_struct_array_and_map_coerced(self) -> None: + schema = StructType( + [ + StructField( + "items", + ArrayType(StructType([StructField("v", DoubleType(), True)])), + True, + ), + StructField("m", MapType(StringType(), DoubleType()), True), + ] + ) + result = coerce_to_schema({"items": [{"v": 1}], "m": {"k": 2}}, schema) + assert result["items"][0]["v"] == 1.0 + assert isinstance(result["items"][0]["v"], float) + assert result["m"]["k"] == 2.0 + + def test_none_and_missing_fields_preserved(self) -> None: + schema = StructType( + [ + StructField("v", DoubleType(), True), + StructField("w", DoubleType(), True), + ] + ) + # `w` absent from the value stays absent (Spark fills null). + result = coerce_to_schema({"v": None}, schema) + assert result == {"v": None} + + +class TestSanitizeRow: + def test_nested_geometry_converted(self) -> None: + row = { + "id": "x", + "nested": {"geometry": "POINT (1 2)"}, + } + result = sanitize_row(row) + assert isinstance(result["nested"]["geometry"], bytes) + + def test_top_level_geometry_converted(self) -> None: + row = {"id": "x", "geometry": "POINT (1 2)"} + result = sanitize_row(row) + assert isinstance(result["geometry"], bytes) + + def test_non_wkt_string_at_geometry_key_unchanged(self) -> None: + row = {"id": "x", "geometry": "not-a-geometry"} + result = sanitize_row(row) + assert result["geometry"] == "not-a-geometry" + + def test_non_geometry_keys_unchanged(self) -> None: + row = {"id": "x", "name": "POINT (1 2)"} + result = sanitize_row(row) + assert result["name"] == "POINT (1 2)" + + +class TestSchemaAssertions: + def test_assert_schema_covers_checks_passes(self, spark: SparkSession) -> None: + schema = StructType( + [ + StructField("id", StringType()), + StructField("x", IntegerType()), + ] + ) + checks = [ + Check( + field="id", + name="required", + expr=F.lit(None), + shape=CheckShape.SCALAR, + read_columns=frozenset({"id"}), + ) + ] + assert_schema_covers_checks(schema, checks) # should not raise + + def test_assert_schema_covers_synthetic_field(self, spark: SparkSession) -> None: + schema = StructType([StructField("sources", ArrayType(StringType()))]) + checks = [ + Check( + field="sources_min_length", + name="min_length", + expr=F.lit(None), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ) + ] + assert_schema_covers_checks(schema, checks) # should not raise + + def test_assert_schema_covers_checks_missing_field( + self, spark: SparkSession + ) -> None: + schema = StructType([StructField("id", StringType())]) + checks = [ + Check( + field="missing", + name="required", + expr=F.lit(None), + shape=CheckShape.SCALAR, + read_columns=frozenset({"missing"}), + ) + ] + with pytest.raises(AssertionError, match="missing"): + assert_schema_covers_checks(schema, checks) + + def test_assert_schema_covers_model_check_columns( + self, spark: SparkSession + ) -> None: + """A model check passes when every column it reads is in the schema.""" + schema = StructType( + [ + StructField("id", StringType()), + StructField("a", StringType()), + StructField("b", StringType()), + ] + ) + checks = [ + Check( + field="radio_group", + name="radio_group", + expr=F.lit(None), + shape=CheckShape.SCALAR, + read_columns=frozenset({"a", "b"}), + ) + ] + assert_schema_covers_checks(schema, checks) # should not raise + + +class TestIndexViolations: + def test_groups_by_scenario_id(self) -> None: + uuid_a = scenario_uuid("f::a:required") + uuid_b = scenario_uuid("f::b:enum") + scenario_map = {uuid_a: "f::a:required", uuid_b: "f::b:enum"} + violation_rows = [ + Row( + _scenario_id=uuid_a, + x=1, + field="a", + check="required", + message="missing", + ), + Row( + _scenario_id=uuid_b, + x=2, + field="b", + check="enum", + message="invalid", + ), + ] + result = index_violations(violation_rows, scenario_map) + assert result["f::a:required"] == {("a", "required")} + assert result["f::b:enum"] == {("b", "enum")} + + def test_multiple_violations_per_scenario(self) -> None: + uuid_a = scenario_uuid("f::a:r") + scenario_map = {uuid_a: "f::a:r"} + violation_rows = [ + Row( + _scenario_id=uuid_a, + x=1, + field="a", + check="required", + message="m1", + ), + Row( + _scenario_id=uuid_a, + x=1, + field="a", + check="bounds", + message="m2", + ), + ] + result = index_violations(violation_rows, scenario_map) + assert result["f::a:r"] == {("a", "required"), ("a", "bounds")} + + def test_empty_violations(self) -> None: + result = index_violations([], {}) + assert result == {} diff --git a/packages/overture-schema-pyspark/tests/test_helpers.py b/packages/overture-schema-pyspark/tests/test_helpers.py new file mode 100644 index 000000000..b202ce3d1 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/test_helpers.py @@ -0,0 +1,147 @@ +"""Tests for the conformance test helpers.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from ._support.helpers import PathTraversalError, deep_merge, set_at_path + + +class TestSetAtPath: + def test_simple_field(self) -> None: + row = {"name": "Alice"} + result = set_at_path("name", "Bob")(row) + assert result["name"] == "Bob" + + def test_does_not_mutate_original(self) -> None: + row = {"name": "Alice"} + set_at_path("name", "Bob")(row) + assert row["name"] == "Alice" + + def test_nested_field(self) -> None: + row = {"outer": {"inner": "old"}} + result = set_at_path("outer.inner", "new")(row) + assert result["outer"]["inner"] == "new" + + def test_array_index_zero(self) -> None: + row = {"items": [{"value": 1}, {"value": 2}]} + result = set_at_path("items[].value", 99)(row) + assert result["items"][0]["value"] == 99 + assert result["items"][1]["value"] == 2 # untouched + + def test_set_to_none(self) -> None: + row = {"country": "US"} + result = set_at_path("country", None)(row) + assert result["country"] is None + + def test_nested_array(self) -> None: + row = {"rules": [{"tags": [{"v": "x"}]}]} + result = set_at_path("rules[].tags[].v", "y")(row) + assert result["rules"][0]["tags"][0]["v"] == "y" + + def test_deep_nested(self) -> None: + row = {"a": {"b": {"c": {"d": "old"}}}} + result = set_at_path("a.b.c.d", "new")(row) + assert result["a"]["b"]["c"]["d"] == "new" + + def test_returns_callable(self) -> None: + mutate = set_at_path("a.b", 1) + assert callable(mutate) + assert mutate({"a": {"b": 0}}) == {"a": {"b": 1}} + + +class TestSetAtPathTraversalErrors: + def test_raises_on_empty_array(self) -> None: + row: dict[str, Any] = {"items": []} + with pytest.raises(PathTraversalError): + set_at_path("items[].value", "x")(row) + + def test_raises_on_empty_nested_array(self) -> None: + row: dict[str, Any] = {"names": {"rules": []}} + with pytest.raises(PathTraversalError): + set_at_path("names.rules[].value", "x")(row) + + def test_error_message_empty_array_names_path(self) -> None: + row: dict[str, Any] = {"names": {"rules": []}} + with pytest.raises(PathTraversalError, match="rules"): + set_at_path("names.rules[].value", "x")(row) + + def test_raises_on_empty_path(self) -> None: + mutator = set_at_path("", "x") + with pytest.raises(PathTraversalError, match="Empty path"): + mutator({}) + + +class TestSetAtPathScaffolding: + def test_null_struct_intermediate_scaffolded(self) -> None: + row = {"id": "x", "names": None} + result = set_at_path("names.primary", "test")(row) + assert result["names"]["primary"] == "test" + + def test_null_array_intermediate_scaffolded(self) -> None: + row = {"id": "x", "rules": None} + result = set_at_path("rules[].value", "test")(row) + assert result["rules"][0]["value"] == "test" + + def test_null_nested_struct_in_array_scaffolded(self) -> None: + row = {"id": "x", "items": [{"nested": None}]} + result = set_at_path("items[].nested.field", "test")(row) + assert result["items"][0]["nested"]["field"] == "test" + + def test_deep_null_chain_scaffolded(self) -> None: + row = {"id": "x", "a": None} + result = set_at_path("a.b[].c", "test")(row) + assert result["a"]["b"][0]["c"] == "test" + + def test_chained_calls_preserve_prior_content(self) -> None: + """Chaining set_at_path preserves values set by prior calls.""" + row = {"items": None} + with_kind = set_at_path("items[].kind", "height")(row) + with_both = set_at_path("items[].value", 5.2)(with_kind) + assert with_both["items"][0]["kind"] == "height" + assert with_both["items"][0]["value"] == 5.2 + + def test_chained_calls_through_deep_null_path(self) -> None: + """Chained calls scaffold and preserve through deeply nested nulls.""" + row = {"outer": None} + with_disc = set_at_path("outer[].inner[].dimension", "height")(row) + with_value = set_at_path("outer[].inner[].value", None)(with_disc) + assert with_value["outer"][0]["inner"][0]["dimension"] == "height" + assert with_value["outer"][0]["inner"][0]["value"] is None + + +class TestDeepMerge: + def test_flat_merge(self) -> None: + base = {"a": 1, "b": 2} + scaffold = {"b": 3, "c": 4} + assert deep_merge(base, scaffold) == {"a": 1, "b": 3, "c": 4} + + def test_nested_dict_merge(self) -> None: + base = {"a": {"x": 1, "y": 2}} + scaffold = {"a": {"y": 3, "z": 4}} + assert deep_merge(base, scaffold) == {"a": {"x": 1, "y": 3, "z": 4}} + + def test_array_replace(self) -> None: + base = {"items": [{"a": 1}]} + scaffold = {"items": [{"b": 2}]} + assert deep_merge(base, scaffold) == {"items": [{"b": 2}]} + + def test_does_not_mutate_base(self) -> None: + base = {"a": {"x": 1}} + scaffold = {"a": {"y": 2}} + result = deep_merge(base, scaffold) + assert "y" not in base["a"] + assert result == {"a": {"x": 1, "y": 2}} + + def test_empty_scaffold(self) -> None: + base = {"a": 1} + assert deep_merge(base, {}) == {"a": 1} + + def test_scaffold_adds_new_key(self) -> None: + base = {"a": 1} + scaffold = {"speed_limits": [{"max_speed": {"value": 60}}]} + result = deep_merge(base, scaffold) + assert result["a"] == 1 + assert result["speed_limits"] == [{"max_speed": {"value": 60}}] diff --git a/packages/overture-schema-pyspark/tests/test_mutations.py b/packages/overture-schema-pyspark/tests/test_mutations.py new file mode 100644 index 000000000..707ab0f41 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/test_mutations.py @@ -0,0 +1,534 @@ +"""Tests for model-level mutation functions.""" + +from typing import Any + +import pytest + +from ._support.helpers import PathTraversalError +from ._support.mutations import ( + _get_nested, + _set_nested, + _walk_strict, + mutate_forbid_if, + mutate_map_key, + mutate_map_value, + mutate_min_fields_set, + mutate_radio_group, + mutate_require_any_of, + mutate_require_any_true, + mutate_require_if, + mutate_unique_items, +) + + +class TestMutateMapKey: + def test_replaces_key_preserving_value(self) -> None: + row = {"names": {"en": "clean"}} + result = mutate_map_key(row, "names", "123") + assert result["names"] == {"123": "clean"} + + def test_nested_path(self) -> None: + row = {"names": {"common": {"en": "clean"}}} + result = mutate_map_key(row, "names.common", "123") + assert result["names"]["common"] == {"123": "clean"} + + def test_does_not_mutate_original(self) -> None: + row = {"names": {"en": "clean"}} + mutate_map_key(row, "names", "123") + assert row["names"] == {"en": "clean"} + + def test_missing_map_raises(self) -> None: + with pytest.raises(PathTraversalError): + mutate_map_key({"other": 1}, "names", "123") + + def test_empty_map_raises(self) -> None: + with pytest.raises(PathTraversalError): + mutate_map_key({"names": {}}, "names", "123") + + +class TestMutateMapValue: + def test_replaces_value_preserving_key(self) -> None: + row = {"names": {"en": "clean"}} + result = mutate_map_value(row, "names", " has spaces ") + assert result["names"] == {"en": " has spaces "} + + def test_nested_path(self) -> None: + row = {"names": {"common": {"en": "clean"}}} + result = mutate_map_value(row, "names.common", " bad ") + assert result["names"]["common"] == {"en": " bad "} + + def test_does_not_mutate_original(self) -> None: + row = {"names": {"en": "clean"}} + mutate_map_value(row, "names", " bad ") + assert row["names"] == {"en": "clean"} + + def test_missing_map_raises(self) -> None: + with pytest.raises(PathTraversalError): + mutate_map_value({"other": 1}, "names", " bad ") + + +class TestMutateRequireAnyOf: + def test_nulls_all_named_fields(self) -> None: + row = {"a": 1, "b": 2, "c": 3} + result = mutate_require_any_of(row, ["a", "b"]) + assert result["a"] is None + assert result["b"] is None + assert result["c"] == 3 + + def test_does_not_mutate_original(self) -> None: + row = {"a": 1, "b": 2} + mutate_require_any_of(row, ["a"]) + assert row["a"] == 1 + + +class TestMutateRadioGroup: + def test_sets_two_fields_to_true(self) -> None: + row = {"is_land": True, "is_territorial": False, "other": "x"} + result = mutate_radio_group(row, ["is_land", "is_territorial"]) + assert result["is_land"] is True + assert result["is_territorial"] is True + + def test_does_not_mutate_original(self) -> None: + row = {"a": False, "b": False} + mutate_radio_group(row, ["a", "b"]) + assert row["a"] is False + + +class TestMutateRequireAnyTrue: + def test_sets_each_field_to_disabling_value(self) -> None: + row = {"is_land": True, "is_territorial": True, "other": "x"} + result = mutate_require_any_true( + row, {"is_land": False, "is_territorial": False} + ) + assert result["is_land"] is False + assert result["is_territorial"] is False + assert result["other"] == "x" + + def test_does_not_mutate_original(self) -> None: + row = {"is_land": True, "is_territorial": True} + mutate_require_any_true(row, {"is_land": False, "is_territorial": False}) + assert row["is_land"] is True + + +class TestMutateMinFieldsSet: + def test_nulls_all_named_fields(self) -> None: + row = {"a": 1, "b": 2, "c": 3} + result = mutate_min_fields_set(row, ["a", "b", "c"]) + assert result["a"] is None + assert result["b"] is None + assert result["c"] is None + + def test_leaves_unlisted_fields_alone(self) -> None: + row = {"a": 1, "b": 2, "other": "keep"} + result = mutate_min_fields_set(row, ["a", "b"]) + assert result["other"] == "keep" + + def test_does_not_mutate_original(self) -> None: + row = {"a": 1, "b": 2} + mutate_min_fields_set(row, ["a", "b"]) + assert row["a"] == 1 + + def test_with_array_path_nulls_inside_each_element(self) -> None: + row = {"items": [{"a": 1, "b": 2}, {"a": 3, "b": 4}]} + result = mutate_min_fields_set(row, ["a", "b"], array_path="items") + assert result["items"] == [{"a": None, "b": None}, {"a": None, "b": None}] + + +class TestMutateRequireIf: + def test_sets_condition_and_nulls_targets(self) -> None: + row = {"subtype": "other", "admin_level": 5} + result = mutate_require_if(row, ["admin_level"], "subtype", "country") + assert result["subtype"] == "country" + assert result["admin_level"] is None + + def test_does_not_mutate_original(self) -> None: + row = {"subtype": "other", "admin_level": 5} + mutate_require_if(row, ["admin_level"], "subtype", "country") + assert row["subtype"] == "other" + + +class TestMutateForbidIf: + def test_sets_condition_and_ensures_non_null(self) -> None: + row = {"subtype": "other", "admin_level": None} + result = mutate_forbid_if(row, ["admin_level"], "subtype", "country") + assert result["subtype"] == "country" + assert result["admin_level"] is not None + + def test_preserves_existing_non_null(self) -> None: + row = {"subtype": "other", "admin_level": 5} + result = mutate_forbid_if(row, ["admin_level"], "subtype", "country") + assert result["admin_level"] == 5 + + def test_uses_fill_value_for_array_field(self) -> None: + row = {"subtype": "other", "destinations": None} + result = mutate_forbid_if( + row, + ["destinations"], + "subtype", + "road", + fill_values={"destinations": [{}]}, + ) + assert result["destinations"] == [{}] + + def test_uses_fill_value_for_struct_field(self) -> None: + row = {"subtype": "other", "road_surface": None} + result = mutate_forbid_if( + row, + ["road_surface"], + "subtype", + "road", + fill_values={"road_surface": {}}, + ) + assert result["road_surface"] == {} + + def test_fill_value_ignored_when_field_already_non_null(self) -> None: + row = {"subtype": "other", "destinations": [{"id": "x"}]} + result = mutate_forbid_if( + row, + ["destinations"], + "subtype", + "road", + fill_values={"destinations": [{}]}, + ) + assert result["destinations"] == [{"id": "x"}] + + +class TestMutateRequireAnyOfNested: + def test_nulls_fields_within_array_elements(self) -> None: + row = { + "items": [ + {"a": 1, "b": 2, "c": 3}, + {"a": 4, "b": 5, "c": 6}, + ] + } + result = mutate_require_any_of(row, ["a", "b"], array_path="items") + for item in result["items"]: + assert item["a"] is None + assert item["b"] is None + assert item["c"] is not None + + def test_nulls_fields_within_nested_struct(self) -> None: + row = { + "items": [ + {"when": {"a": 1, "b": 2}}, + ] + } + result = mutate_require_any_of( + row, ["a", "b"], array_path="items", struct_path="when" + ) + assert result["items"][0]["when"]["a"] is None + assert result["items"][0]["when"]["b"] is None + + def test_creates_stub_element_when_array_is_null(self) -> None: + row = {"items": None} + result = mutate_require_any_of(row, ["a", "b"], array_path="items") + assert isinstance(result["items"], list) + assert len(result["items"]) == 1 + assert result["items"][0]["a"] is None + assert result["items"][0]["b"] is None + + def test_creates_stub_with_struct_path_when_null(self) -> None: + row = {"items": None} + result = mutate_require_any_of( + row, ["a", "b"], array_path="items", struct_path="when" + ) + assert result["items"][0]["when"]["a"] is None + assert result["items"][0]["when"]["b"] is None + + def test_does_not_mutate_original(self) -> None: + row = {"items": [{"a": 1, "b": 2}]} + mutate_require_any_of(row, ["a", "b"], array_path="items") + assert row["items"][0]["a"] == 1 + + +class TestMutateMapValueModelConstraint: + """`map_path` threads a model mutation into a `dict[K, Model]` value model. + + A model-level constraint on a map's value model targets the value, not + the row root. `map_path` names the map column; the mutation corrupts the + single entry's value (stubbing one when the map is absent), so the + generated `::invalid` row actually trips the value-model constraint. + """ + + def test_require_any_of_nulls_fields_in_map_value(self) -> None: + row = {"subs": {"en": {"foo": 1, "bar": "x"}}} + result = mutate_require_any_of(row, ["foo", "bar"], map_path="subs") + assert result["subs"]["en"] == {"foo": None, "bar": None} + + def test_require_any_of_preserves_map_key(self) -> None: + row = {"subs": {"en": {"foo": 1, "bar": "x"}}} + result = mutate_require_any_of(row, ["foo", "bar"], map_path="subs") + assert list(result["subs"]) == ["en"] + + def test_require_any_of_stubs_missing_map(self) -> None: + row: dict = {"subs": None} + result = mutate_require_any_of(row, ["foo", "bar"], map_path="subs") + assert isinstance(result["subs"], dict) and result["subs"] + assert next(iter(result["subs"].values())) == {"foo": None, "bar": None} + + def test_struct_path_descends_into_map_value(self) -> None: + row = {"subs": {"en": {"inner": {"foo": 1, "bar": 2}}}} + result = mutate_require_any_of( + row, ["foo", "bar"], map_path="subs", struct_path="inner" + ) + assert result["subs"]["en"]["inner"] == {"foo": None, "bar": None} + + def test_nested_map_column_path(self) -> None: + row = {"outer": {"subs": {"en": {"foo": 1, "bar": 2}}}} + result = mutate_require_any_of(row, ["foo", "bar"], map_path="outer.subs") + assert result["outer"]["subs"]["en"] == {"foo": None, "bar": None} + + def test_require_any_of_does_not_mutate_original(self) -> None: + row = {"subs": {"en": {"foo": 1, "bar": 2}}} + mutate_require_any_of(row, ["foo", "bar"], map_path="subs") + assert row["subs"]["en"]["foo"] == 1 + + def test_min_fields_set_nulls_fields_in_map_value(self) -> None: + row = {"subs": {"en": {"a": 1, "b": 2}}} + result = mutate_min_fields_set(row, ["a", "b"], map_path="subs") + assert result["subs"]["en"] == {"a": None, "b": None} + + def test_require_if_sets_condition_and_nulls_in_map_value(self) -> None: + row = {"subs": {"en": {"subtype": "other", "admin_level": 5}}} + result = mutate_require_if( + row, ["admin_level"], "subtype", "country", map_path="subs" + ) + value = result["subs"]["en"] + assert value["subtype"] == "country" + assert value["admin_level"] is None + + def test_require_if_stubs_missing_map(self) -> None: + row: dict = {"subs": None} + result = mutate_require_if( + row, ["admin_level"], "subtype", "country", map_path="subs" + ) + value = next(iter(result["subs"].values())) + assert value["subtype"] == "country" + assert value["admin_level"] is None + + def test_forbid_if_sets_condition_and_ensures_non_null_in_map_value(self) -> None: + row = {"subs": {"en": {"subtype": "other", "admin_level": None}}} + result = mutate_forbid_if( + row, ["admin_level"], "subtype", "country", map_path="subs" + ) + value = result["subs"]["en"] + assert value["subtype"] == "country" + assert value["admin_level"] is not None + + +class TestMutateForbidIfNegate: + def test_negate_changes_condition_value(self) -> None: + """negate=True sets condition_field to something != condition_value.""" + row = {"subtype": "road", "destinations": [{"id": "x"}]} + result = mutate_forbid_if(row, ["destinations"], "subtype", "road", negate=True) + assert result["subtype"] != "road" + assert result["destinations"] is not None + + def test_negate_preserves_non_matching_value(self) -> None: + """When condition_field already != condition_value, leave it.""" + row = {"subtype": "water", "class": "canal"} + result = mutate_forbid_if(row, ["class"], "subtype", "road", negate=True) + assert result["subtype"] == "water" + + +class TestMutateRequireIfNegate: + def test_negate_changes_condition_value(self) -> None: + """negate=True sets condition_field to something != condition_value.""" + row = {"subtype": "road", "class": "motorway"} + result = mutate_require_if(row, ["class"], "subtype", "road", negate=True) + assert result["subtype"] != "road" + assert result["class"] is None + + def test_negate_preserves_non_matching_value(self) -> None: + """When condition_field already != condition_value, leave it.""" + row = {"subtype": "water", "class": "canal"} + result = mutate_require_if(row, ["class"], "subtype", "road", negate=True) + assert result["subtype"] == "water" + assert result["class"] is None + + +class TestMutateUniqueItems: + def test_duplicates_first_element(self) -> None: + row = {"ids": [{"value": "a"}, {"value": "b"}]} + result = mutate_unique_items(row, "ids") + assert result["ids"][0] == result["ids"][1] + assert len(result["ids"]) == 3 + + def test_nested_path(self) -> None: + row = {"outer": {"ids": [{"v": 1}, {"v": 2}]}} + result = mutate_unique_items(row, "outer.ids") + assert result["outer"]["ids"][0] == result["outer"]["ids"][1] + + def test_does_not_mutate_original(self) -> None: + row = {"ids": [{"value": "a"}, {"value": "b"}]} + mutate_unique_items(row, "ids") + assert len(row["ids"]) == 2 + + def test_bracket_path_enters_array_element(self) -> None: + row = {"restrictions": [{"when": {"mode": [{"type": "car"}, {"type": "bus"}]}}]} + result = mutate_unique_items(row, "restrictions[].when.mode") + mode = result["restrictions"][0]["when"]["mode"] + assert mode[0] == mode[1] + assert len(mode) == 3 + + def test_empty_array_returns_unchanged(self) -> None: + row: dict = {"items": []} + result = mutate_unique_items(row, "items") + assert result["items"] == [] + + def test_none_array_raises_traversal_error(self) -> None: + row: dict = {"ids": None} + with pytest.raises(PathTraversalError): + mutate_unique_items(row, "ids") + + def test_missing_key_raises_traversal_error(self) -> None: + row: dict = {"other": "x"} + with pytest.raises(PathTraversalError): + mutate_unique_items(row, "missing.nested") + + def test_nested_bracket_deep(self) -> None: + """Two levels of bracket nesting.""" + row: dict = {"outer": [{"inner": [{"vals": [{"x": 1}]}]}]} + result = mutate_unique_items(row, "outer[].inner[].vals") + vals = result["outer"][0]["inner"][0]["vals"] + assert vals[0] == vals[1] + + def test_terminal_bracket_duplicates_inner_list(self) -> None: + """Terminal `[]` targets the inner list at element 0 of the named field.""" + row: dict = {"hierarchies": [[{"a": 1}]]} + result = mutate_unique_items(row, "hierarchies[]") + inner = result["hierarchies"][0] + assert inner[0] == inner[1] + assert len(inner) == 2 + + def test_terminal_bracket_non_list_inner_raises(self) -> None: + """Terminal `[]` with non-list content at element 0 raises.""" + row: dict = {"hierarchies": [{"a": 1}]} + with pytest.raises(PathTraversalError): + mutate_unique_items(row, "hierarchies[]") + + +class TestWalkStrict: + def test_simple_struct(self) -> None: + row = {"a": {"b": {"c": 42}}} + result = _walk_strict(row, "a.b") + assert result == {"c": 42} + + def test_root_returns_row(self) -> None: + row = {"x": 1} + assert _walk_strict(row, "") == row + + def test_missing_key_raises(self) -> None: + row = {"a": {"b": 1}} + with pytest.raises(PathTraversalError, match="Missing"): + _walk_strict(row, "a.c") + + def test_null_intermediate_raises(self) -> None: + row = {"a": None} + with pytest.raises(PathTraversalError, match="a"): + _walk_strict(row, "a.b") + + def test_error_message_includes_segment_name(self) -> None: + row = {"outer": {"inner": None}} + with pytest.raises(PathTraversalError, match="inner"): + _walk_strict(row, "outer.inner.leaf") + + def test_error_message_includes_full_path(self) -> None: + row = {"outer": None} + with pytest.raises(PathTraversalError, match="outer.inner"): + _walk_strict(row, "outer.inner") + + def test_array_segment_descends_to_element_zero(self) -> None: + row = {"items": [{"val": 5}]} + result = _walk_strict(row, "items[]") + assert result == {"val": 5} + + def test_array_segment_empty_raises(self) -> None: + row: dict[str, Any] = {"items": []} + with pytest.raises(PathTraversalError, match="items"): + _walk_strict(row, "items[]") + + def test_array_segment_with_struct_after(self) -> None: + row = {"rules": [{"when": {"mode": [{"type": "car"}]}}]} + result = _walk_strict(row, "rules[].when") + assert result == {"mode": [{"type": "car"}]} + + def test_nested_list_descends_each_bracket_level(self) -> None: + row = {"grid": [[{"val": 7}]]} + result = _walk_strict(row, "grid[][]") + assert result == {"val": 7} + + def test_nested_list_empty_inner_raises(self) -> None: + row: dict[str, Any] = {"grid": [[]]} + with pytest.raises(PathTraversalError, match="grid"): + _walk_strict(row, "grid[][]") + + +class TestGetNested: + def test_simple_lookup(self) -> None: + row = {"a": {"b": 3}} + assert _get_nested(row, "a.b") == 3 + + def test_missing_key_returns_none(self) -> None: + row = {"a": 1} + assert _get_nested(row, "b") is None + + def test_missing_nested_key_returns_none(self) -> None: + row = {"a": {"b": 1}} + assert _get_nested(row, "a.c") is None + + def test_none_intermediate_returns_none(self) -> None: + row = {"a": None} + assert _get_nested(row, "a.b") is None + + def test_non_dict_intermediate_returns_none(self) -> None: + row = {"a": [1, 2, 3]} + assert _get_nested(row, "a.b") is None + + def test_rejects_array_path(self) -> None: + with pytest.raises(ValueError, match="struct-only"): + _get_nested({"items": [{"v": 1}]}, "items[].v") + + +class TestSetNested: + def test_set_simple_field(self) -> None: + d = {"a": 1} + _set_nested(d, "a", 2) + assert d["a"] == 2 + + def test_set_nested_field(self) -> None: + d = {"outer": {"inner": "old"}} + _set_nested(d, "outer.inner", "new") + assert d["outer"]["inner"] == "new" + + def test_null_value_through_none_intermediate_silent(self) -> None: + """Nulling through a None intermediate is a no-op — already null.""" + d = {"a": None} + _set_nested(d, "a.b", None) + assert d["a"] is None + + def test_null_value_through_missing_intermediate_silent(self) -> None: + d: dict = {} + _set_nested(d, "a.b", None) + assert "a" not in d + + def test_non_null_through_none_intermediate_raises_path_traversal_error( + self, + ) -> None: + d = {"a": None} + with pytest.raises(PathTraversalError, match="a"): + _set_nested(d, "a.b", "value") + + def test_create_scaffolds_missing_intermediate(self) -> None: + d: dict = {} + _set_nested(d, "a.b", "v", create=True) + assert d["a"]["b"] == "v" + + def test_create_scaffolds_none_intermediate(self) -> None: + d: dict = {"a": None} + _set_nested(d, "a.b", "v", create=True) + assert d["a"]["b"] == "v" + + def test_rejects_array_path(self) -> None: + with pytest.raises(ValueError, match="struct-only"): + _set_nested({"items": []}, "items[].v", "x") diff --git a/packages/overture-schema-pyspark/tests/test_validate.py b/packages/overture-schema-pyspark/tests/test_validate.py new file mode 100644 index 000000000..9539b8977 --- /dev/null +++ b/packages/overture-schema-pyspark/tests/test_validate.py @@ -0,0 +1,869 @@ +"""Tests for validation pipeline.""" + +import re +from collections.abc import Iterator + +import pytest +from overture.schema.pyspark._registry import REGISTRY +from overture.schema.pyspark.check import Check, CheckShape +from overture.schema.pyspark.expressions.column_patterns import map_values_check +from overture.schema.pyspark.validate import ( + ValidationResult, + _normalize_suppress, + evaluate_checks, + explain_errors, + filter_errors, + model_keys, + model_names, + validate_model, +) +from pyspark.sql import DataFrame, Row, SparkSession +from pyspark.sql import functions as F +from pyspark.sql.types import ( + ArrayType, + DoubleType, + IntegerType, + MapType, + StringType, + StructField, + StructType, +) + +from ._support.registry import register_model + + +def _scalar_check( + field: str, name: str, expr: F.Column, *, read_columns: frozenset[str] | None = None +) -> Check: + return Check( + field=field, + name=name, + expr=expr, + shape=CheckShape.SCALAR, + read_columns=read_columns if read_columns is not None else frozenset({field}), + ) + + +def _array_check( + field: str, name: str, expr: F.Column, *, read_columns: frozenset[str] | None = None +) -> Check: + return Check( + field=field, + name=name, + expr=expr, + shape=CheckShape.ARRAY, + read_columns=read_columns if read_columns is not None else frozenset({field}), + ) + + +def _row(**kwargs: object) -> Row: + """Build a row with convenience id/theme/type defaults.""" + defaults: dict[str, object] = {"id": "id1", "theme": "t", "type": "f"} + defaults.update(kwargs) + return Row(**defaults) + + +class TestEvaluateChecks: + """Tests for evaluate_checks().""" + + def test_appends_error_columns(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [_scalar_check("value", "required", F.lit("fail"))] + evaluated = evaluate_checks(df, checks) + assert "_err_0" in evaluated.columns + assert set(df.columns) < set(evaluated.columns) + + def test_multiple_checks(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [ + _scalar_check("a", "c1", F.lit("e1")), + _scalar_check("b", "c2", F.lit("e2")), + ] + evaluated = evaluate_checks(df, checks) + assert "_err_0" in evaluated.columns + assert "_err_1" in evaluated.columns + + def test_error_column_is_array_string(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [_scalar_check("value", "required", F.lit("fail"))] + evaluated = evaluate_checks(df, checks) + row = evaluated.collect()[0] + assert row["_err_0"] == ["fail"] + + def test_null_error_for_passing_check(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [_scalar_check("value", "ok", F.lit(None).cast("string"))] + evaluated = evaluate_checks(df, checks) + row = evaluated.collect()[0] + assert row["_err_0"] == [] + + +class TestFilterErrors: + """Tests for filter_errors().""" + + def test_keeps_failing_rows(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [_scalar_check("value", "required", F.lit("fail"))] + evaluated = evaluate_checks(df, checks) + result = filter_errors(evaluated, checks) + assert result.count() == 1 + + def test_removes_passing_rows(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [_scalar_check("value", "ok", F.lit(None).cast("string"))] + evaluated = evaluate_checks(df, checks) + result = filter_errors(evaluated, checks) + assert result.count() == 0 + + def test_strips_error_columns(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [_scalar_check("value", "required", F.lit("fail"))] + evaluated = evaluate_checks(df, checks) + result = filter_errors(evaluated, checks) + assert not any(c.startswith("_err_") for c in result.columns) + assert set(result.columns) == set(df.columns) + + def test_preserves_schema(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [_scalar_check("value", "required", F.lit("fail"))] + evaluated = evaluate_checks(df, checks) + result = filter_errors(evaluated, checks) + assert result.schema == df.schema + + def test_mixed_rows(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row(id="pass"), _row(id="fail")]) + checks = [ + _scalar_check( + "id", + "not_fail", + F.when(F.col("id") == "fail", F.lit("bad")), + ), + ] + evaluated = evaluate_checks(df, checks) + result = filter_errors(evaluated, checks) + assert result.count() == 1 + assert result.collect()[0]["id"] == "fail" + + +class TestExplainErrors: + """Tests for explain_errors().""" + + def test_scalar_violation(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [_scalar_check("value", "required", F.lit("missing"))] + evaluated = evaluate_checks(df, checks) + result = explain_errors(evaluated, checks) + rows = result.collect() + assert len(rows) == 1 + assert rows[0]["field"] == "value" + assert rows[0]["check"] == "required" + assert rows[0]["message"] == "missing" + + def test_array_violation(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [_array_check("arr", "elem", F.array(F.lit("e1"), F.lit("e2")))] + evaluated = evaluate_checks(df, checks) + result = explain_errors(evaluated, checks) + messages = sorted(r["message"] for r in result.collect()) + assert messages == ["e1", "e2"] + + def test_no_violations(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [_scalar_check("value", "ok", F.lit(None).cast("string"))] + evaluated = evaluate_checks(df, checks) + result = explain_errors(evaluated, checks) + assert result.count() == 0 + + def test_preserves_original_columns(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [_scalar_check("value", "required", F.lit("fail"))] + evaluated = evaluate_checks(df, checks) + result = explain_errors(evaluated, checks) + rows = result.collect() + assert rows[0]["id"] == "id1" + assert set(result.columns) == {*df.columns, "field", "check", "message"} + + def test_output_columns(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + checks = [_scalar_check("x", "required", F.lit("err"))] + evaluated = evaluate_checks(df, checks) + result = explain_errors(evaluated, checks) + expected_cols = {*df.columns, "field", "check", "message"} + assert set(result.columns) == expected_cols + + def test_empty_checks_returns_empty_dataframe_with_schema( + self, spark: SparkSession + ) -> None: + # Regression: explain_errors([]) on rows with no checks must + # return a typed empty DataFrame, not invoke `stack(0, ...)` + # (which Spark rejects). Consumers expect the standard + # `field/check/message` columns even when nothing fired. + df = spark.createDataFrame([_row()]) + result = explain_errors(df, []) + assert result.count() == 0 + assert set(result.columns) == {*df.columns, "field", "check", "message"} + + +class TestUserErrColumn: + """`_err_` is reserved; user `_err_*` names are passed through.""" + + def test_user_err_named_column_preserved(self, spark: SparkSession) -> None: + # Regression: `_orig_columns` strips only `_err_`. A + # user-supplied column like `_err_foo` must survive + # filter_errors / explain_errors round-trips. + df = spark.createDataFrame([_row(_err_foo="custom-data")]) + checks = [_scalar_check("value", "required", F.lit("fail"))] + evaluated = evaluate_checks(df, checks) + filtered = filter_errors(evaluated, checks) + assert "_err_foo" in filtered.columns + assert filtered.collect()[0]["_err_foo"] == "custom-data" + + explained = explain_errors(evaluated, checks) + assert "_err_foo" in explained.columns + assert explained.collect()[0]["_err_foo"] == "custom-data" + + +class TestReservedColumnCollisions: + """Working/output columns must not collide with user input columns. + + `evaluate_checks` appends `_err_` columns; `explain_errors` + materializes `_idx`/`_errors` scratch columns and emits its + `field`/`check`/`message` output. An input column sharing any of + these names yields duplicate attributes -> AMBIGUOUS_REFERENCE (or + silent loss via the `_err_` strip), so both entry points reject + the collision up front with a clear error. + """ + + def test_evaluate_checks_rejects_err_column(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row(_err_0="dup")]) + checks = [_scalar_check("value", "required", F.lit("fail"))] + with pytest.raises(ValueError, match=r"_err_0.*rename or drop"): + evaluate_checks(df, checks) + + def test_evaluate_checks_allows_non_digit_err_column( + self, spark: SparkSession + ) -> None: + # Only `_err_` is reserved; `_err_foo` is a user column. + df = spark.createDataFrame([_row(_err_foo="ok")]) + checks = [_scalar_check("value", "required", F.lit("fail"))] + evaluated = evaluate_checks(df, checks) + assert "_err_foo" in evaluated.columns + + def test_evaluate_checks_rejects_reevaluation(self, spark: SparkSession) -> None: + # The realistic trigger: a persisted `result.evaluated` (which + # carries `_err_0..N`) fed back through validation. + df = spark.createDataFrame([_row()]) + checks = [_scalar_check("value", "required", F.lit("fail"))] + evaluated = evaluate_checks(df, checks) + with pytest.raises(ValueError, match="_err_0"): + evaluate_checks(evaluated, checks) + + @pytest.mark.parametrize( + "reserved", ["_idx", "_errors", "field", "check", "message"] + ) + def test_explain_errors_rejects_reserved_input_column( + self, spark: SparkSession, reserved: str + ) -> None: + df = spark.createDataFrame([_row(**{reserved: "dup"})]) + checks = [_scalar_check("value", "required", F.lit("fail"))] + evaluated = evaluate_checks(df, checks) + with pytest.raises(ValueError, match=re.escape(reserved)): + explain_errors(evaluated, checks) + + def test_explain_errors_rejects_reserved_with_no_checks( + self, spark: SparkSession + ) -> None: + # The n == 0 branch also emits field/check/message, so the guard + # must precede it. + df = spark.createDataFrame([_row(field="dup")]) + with pytest.raises(ValueError, match="field"): + explain_errors(df, []) + + +class TestSinglePassPipeline: + """Tests for the evaluate-once pattern used by the CLI.""" + + def test_shared_evaluated_gives_same_results(self, spark: SparkSession) -> None: + """filter_errors + explain_errors from the same evaluated DataFrame.""" + df = spark.createDataFrame([_row(id="ok"), _row(id="bad")]) + checks = [ + _scalar_check( + "id", + "not_bad", + F.when(F.col("id") == "bad", F.lit("is bad")), + ), + ] + evaluated = evaluate_checks(df, checks) + filtered = filter_errors(evaluated, checks) + explained = explain_errors(evaluated, checks) + assert filtered.count() == 1 + assert filtered.collect()[0]["id"] == "bad" + assert explained.count() == 1 + assert explained.collect()[0]["field"] == "id" + + +class TestNormalizeSuppress: + def test_empty(self) -> None: + roots, pairs = _normalize_suppress(()) + assert roots == set() + assert pairs == set() + + def test_bare_strings(self) -> None: + roots, pairs = _normalize_suppress(["sources", "theme"]) + assert roots == {"sources", "theme"} + assert pairs == set() + + def test_tuples(self) -> None: + roots, pairs = _normalize_suppress([("sources[].confidence", "bounds")]) + assert roots == set() + assert pairs == {("sources[].confidence", "bounds")} + + def test_check_objects(self, spark: SparkSession) -> None: + check = Check( + field="radio_group", + name="radio_group", + expr=F.lit(None), + shape=CheckShape.SCALAR, + read_columns=frozenset(), + ) + roots, pairs = _normalize_suppress([check]) + assert roots == set() + assert pairs == {("radio_group", "radio_group")} + + def test_mixed(self, spark: SparkSession) -> None: + check = Check( + field="radio_group", + name="radio_group", + expr=F.lit(None), + shape=CheckShape.SCALAR, + read_columns=frozenset(), + ) + roots, pairs = _normalize_suppress( + [ + "sources", + ("theme", "enum"), + check, + ] + ) + assert roots == {"sources"} + assert pairs == {("theme", "enum"), ("radio_group", "radio_group")} + + +# These exercise the populated REGISTRY built by runtime discovery, so they +# require generated expression modules to be present on disk. When the +# generated tree is absent (e.g. a fresh checkout before `make +# generate-pyspark`), the registry is empty and these assertions can't hold. +_requires_generated = pytest.mark.skipif( + not REGISTRY, reason="requires generated expression modules" +) + + +@_requires_generated +def test_model_names_includes_aliases() -> None: + result = model_names() + assert isinstance(result, list) + assert result == sorted(result) + assert "building" in result + assert "segment" in result + assert "overture.schema.buildings:Building" in result + + +@_requires_generated +def test_model_keys_only_canonical() -> None: + result = model_keys() + assert isinstance(result, list) + assert result == sorted(result) + assert "overture.schema.buildings:Building" in result + assert "building" not in result + + +class TestValidationResult: + def test_error_rows_delegates_to_filter_errors(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row(id="ok"), _row(id="bad")]) + checks = [ + _scalar_check( + "id", + "not_bad", + F.when(F.col("id") == "bad", F.lit("is bad")), + ), + ] + evaluated = evaluate_checks(df, checks) + result = ValidationResult( + evaluated=evaluated, + checks=checks, + schema_mismatches=[], + suppressed_checks=[], + ) + error_rows = result.error_rows() + assert error_rows.count() == 1 + assert error_rows.collect()[0]["id"] == "bad" + assert not any(c.startswith("_err_") for c in error_rows.columns) + + def test_frozen(self) -> None: + result = ValidationResult( + evaluated=None, # type: ignore[arg-type] + checks=[], + schema_mismatches=[], + suppressed_checks=[], + ) + with pytest.raises(AttributeError): + result.checks = [] # type: ignore[misc] + + +_VF_TYPE = "_test_validate_feature" +_VF_NESTED_TYPE = "_test_validate_nested" +_VF_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("theme", StringType(), True), + StructField("type", StringType(), True), + StructField("value", StringType(), True), + StructField("sources", StringType(), True), + ] +) +_VF_NESTED_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField( + "bbox", + StructType( + [ + StructField("xmin", StringType(), True), + StructField("xmax", StringType(), True), + ] + ), + True, + ), + ] +) + + +# `_VF_ARRAY_*` exercises a missing field *inside* an array element struct. +# `compare_schemas` encodes the array step as `sources[].confidence`; the +# root-derivation in validate_model must strip the `[]` marker so the +# dropped-check root matches the check's read column (`sources`). +_VF_ARRAY_TYPE = "_test_validate_array_nested" +_VF_ARRAY_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField( + "sources", + ArrayType( + StructType( + [ + StructField("dataset", StringType(), True), + StructField("confidence", DoubleType(), True), + ] + ) + ), + True, + ), + ] +) + + +def _vf_array_checks() -> list[Check]: + return [ + Check( + field="sources", + name="confidence_bounds", + expr=F.transform( + "sources", + lambda el: F.when(el["confidence"] > 1.0, F.lit("confidence too high")), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"sources"}), + ), + ] + + +# `_VF_MODEL_*` exercises a model-level constraint that reads several columns +# directly. When any column it reads is skipped/absent, the exclusion filter +# must drop the model check too. +_VF_MODEL_TYPE = "_test_validate_model_constraint" +_VF_MODEL_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("primary_name", StringType(), True), + StructField("alt_name", StringType(), True), + ] +) + + +def _vf_model_checks() -> list[Check]: + return [ + Check( + field="require_any_of", + name="require_any_of", + expr=F.when( + F.col("primary_name").isNull() & F.col("alt_name").isNull(), + F.lit("at least one name required"), + ), + shape=CheckShape.SCALAR, + read_columns=frozenset({"primary_name", "alt_name"}), + ), + ] + + +# `_VF_MAP_*` exercises a map key/value check, whose expression dereferences +# the map column by name (`map_values_check("license_priority", ...)`). Skipping +# or omitting that column must drop the check, mirroring the array path, rather +# than leaving an unresolvable map projection behind. +_VF_MAP_TYPE = "_test_validate_map_check" +_VF_MAP_SCHEMA = StructType( + [ + StructField("id", StringType(), True), + StructField("license_priority", MapType(StringType(), IntegerType()), True), + ] +) + + +def _vf_map_checks() -> list[Check]: + return [ + Check( + field="license_priority{value}", + name="bounds", + expr=map_values_check( + "license_priority", lambda v: F.when(v < 0, F.lit("negative")) + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"license_priority"}), + ), + ] + + +def _vf_checks() -> list[Check]: + return [ + Check( + field="theme", + name="enum", + expr=F.when(F.col("theme") != "test", F.lit("bad theme")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"theme"}), + ), + Check( + field="value", + name="required", + expr=F.when(F.col("value").isNull(), F.lit("missing")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"value"}), + ), + Check( + field="sources_min_length", + name="min_length", + expr=F.when(F.length("sources") < 1, F.lit("too short")), + shape=CheckShape.SCALAR, + read_columns=frozenset({"sources"}), + ), + ] + + +class TestValidateFeature: + @pytest.fixture(autouse=True) + def _register_vf_type(self) -> Iterator[None]: + with register_model(_VF_TYPE, _VF_SCHEMA, _vf_checks): + yield + + @pytest.fixture() + def _register_nested_type(self) -> Iterator[None]: + with register_model(_VF_NESTED_TYPE, _VF_NESTED_SCHEMA, lambda: []): + yield + + @pytest.fixture() + def _register_map_type(self) -> Iterator[None]: + with register_model(_VF_MAP_TYPE, _VF_MAP_SCHEMA, _vf_map_checks): + yield + + @pytest.fixture() + def _register_array_type(self) -> Iterator[None]: + with register_model(_VF_ARRAY_TYPE, _VF_ARRAY_SCHEMA, _vf_array_checks): + yield + + @pytest.fixture() + def _register_model_type(self) -> Iterator[None]: + with register_model(_VF_MODEL_TYPE, _VF_MODEL_SCHEMA, _vf_model_checks): + yield + + @pytest.fixture() + def vf_df(self, spark: SparkSession) -> DataFrame: + return spark.createDataFrame( + [Row(id="1", theme="test", type=_VF_TYPE, value="ok", sources="s")], + schema=_VF_SCHEMA, + ) + + def test_unknown_type_raises_value_error(self, spark: SparkSession) -> None: + df = spark.createDataFrame([_row()]) + with pytest.raises( + ValueError, match="Unknown entry-point alias.*nonexistent_type_xyz" + ): + validate_model(df, "nonexistent_type_xyz") + + def test_basic_validation(self, vf_df: DataFrame) -> None: + result = validate_model(vf_df, _VF_TYPE) + assert isinstance(result, ValidationResult) + assert result.schema_mismatches == [] + assert len(result.checks) == 3 + assert result.error_rows().count() == 0 + + def test_skip_columns_errors_if_present(self, vf_df: DataFrame) -> None: + with pytest.raises(ValueError, match="skip_columns.*theme.*present"): + validate_model(vf_df, _VF_TYPE, skip_columns=["theme"]) + + def test_skip_columns_filters_checks(self, spark: SparkSession) -> None: + schema_no_theme = StructType( + [f for f in _VF_SCHEMA.fields if f.name != "theme"] + ) + df = spark.createDataFrame( + [Row(id="1", type=_VF_TYPE, value="ok", sources="s")], + schema=schema_no_theme, + ) + result = validate_model(df, _VF_TYPE, skip_columns=["theme"]) + check_fields = [c.field for c in result.checks] + assert "theme" not in check_fields + assert "value" in check_fields + + def test_skip_columns_filters_schema_mismatches(self, spark: SparkSession) -> None: + schema_no_theme = StructType( + [f for f in _VF_SCHEMA.fields if f.name != "theme"] + ) + df = spark.createDataFrame( + [Row(id="1", type=_VF_TYPE, value="ok", sources="s")], + schema=schema_no_theme, + ) + result = validate_model(df, _VF_TYPE, skip_columns=["theme"]) + mismatch_fields = [m.path for m in result.schema_mismatches] + assert "theme" not in mismatch_fields + + def test_ignore_extra_columns(self, spark: SparkSession) -> None: + schema_extra = StructType( + _VF_SCHEMA.fields + [StructField("extra_score", StringType(), True)] + ) + df = spark.createDataFrame( + [ + Row( + id="1", + theme="test", + type=_VF_TYPE, + value="ok", + sources="s", + extra_score="9", + ) + ], + schema=schema_extra, + ) + result = validate_model(df, _VF_TYPE, ignore_extra_columns=["extra_score"]) + mismatch_paths = [m.path for m in result.schema_mismatches] + assert "extra_score" not in mismatch_paths + + def test_suppress_unknown_root_raises(self, vf_df: DataFrame) -> None: + with pytest.raises(ValueError, match="unknown root fields.*typo_field"): + validate_model(vf_df, _VF_TYPE, suppress=["typo_field"]) + + def test_suppress_unknown_pair_raises(self, vf_df: DataFrame) -> None: + with pytest.raises(ValueError, match=r"unknown \(field, name\) pairs"): + validate_model(vf_df, _VF_TYPE, suppress=[("theme", "wrong_name")]) + + def test_suppress_mixed_unknown_lists_both(self, vf_df: DataFrame) -> None: + with pytest.raises(ValueError, match="unknown root fields.*unknown"): + validate_model( + vf_df, + _VF_TYPE, + suppress=["typo_field", ("theme", "wrong_name")], + ) + + def test_suppress_bare_string(self, vf_df: DataFrame) -> None: + result = validate_model(vf_df, _VF_TYPE, suppress=["sources"]) + check_fields = [c.field for c in result.checks] + assert not any(f.startswith("sources") for f in check_fields) + assert len(result.suppressed_checks) == 1 + assert result.suppressed_checks[0].field == "sources_min_length" + + def test_suppress_tuple(self, vf_df: DataFrame) -> None: + result = validate_model(vf_df, _VF_TYPE, suppress=[("value", "required")]) + check_fields_names = [(c.field, c.name) for c in result.checks] + assert ("value", "required") not in check_fields_names + assert len(result.suppressed_checks) == 1 + + def test_suppress_check_object(self, vf_df: DataFrame) -> None: + initial = validate_model(vf_df, _VF_TYPE) + target = [c for c in initial.checks if c.name == "required"][0] + result = validate_model(vf_df, _VF_TYPE, suppress=[target]) + # Column objects can't be compared with ==, so compare by (field, name) + result_pairs = [(c.field, c.name) for c in result.checks] + suppressed_pairs = [(c.field, c.name) for c in result.suppressed_checks] + assert (target.field, target.name) not in result_pairs + assert (target.field, target.name) in suppressed_pairs + + def test_evaluated_has_err_columns(self, vf_df: DataFrame) -> None: + result = validate_model(vf_df, _VF_TYPE) + err_cols = [c for c in result.evaluated.columns if c.startswith("_err_")] + assert len(err_cols) == len(result.checks) + + def test_suppressed_checks_not_in_checks(self, vf_df: DataFrame) -> None: + result = validate_model(vf_df, _VF_TYPE, suppress=[("theme", "enum")]) + for sc in result.suppressed_checks: + assert sc not in result.checks + + def test_all_checks_suppressed(self, vf_df: DataFrame) -> None: + result = validate_model( + vf_df, + _VF_TYPE, + suppress=["theme", "value", "sources"], + ) + assert result.checks == [] + assert result.error_rows().count() == 0 + + def test_missing_column_does_not_raise(self, spark: SparkSession) -> None: + # A DataFrame missing a required column causes AnalysisException when + # evaluate_checks references that column. validate_model must detect + # structurally absent columns via schema_mismatches and silently drop + # the corresponding checks before calling evaluate_checks -- mirroring + # the skip_columns path. + schema_no_theme = StructType( + [f for f in _VF_SCHEMA.fields if f.name != "theme"] + ) + df = spark.createDataFrame( + [Row(id="1", type=_VF_TYPE, value="ok", sources="s")], + schema=schema_no_theme, + ) + result = validate_model(df, _VF_TYPE) + # Must not raise -- returns normally + assert isinstance(result, ValidationResult) + # Missing column is reported as a schema mismatch + mismatch_paths = [m.path for m in result.schema_mismatches] + assert "theme" in mismatch_paths + # No kept check may read the absent column + assert all("theme" not in c.read_columns for c in result.checks) + # Absent-column checks are silently dropped, not tracked in suppressed + assert all("theme" not in c.read_columns for c in result.suppressed_checks) + + def test_absent_columns_exposed_on_result(self, spark: SparkSession) -> None: + # validate_model must expose absent_columns as an ordered tuple so + # callers (e.g. CLI) don't need to re-derive it from schema_mismatches. + schema_no_theme = StructType( + [f for f in _VF_SCHEMA.fields if f.name != "theme"] + ) + df = spark.createDataFrame( + [Row(id="1", type=_VF_TYPE, value="ok", sources="s")], + schema=schema_no_theme, + ) + result = validate_model(df, _VF_TYPE) + assert result.absent_columns == ("theme",) + + def test_absent_columns_empty_when_schema_matches(self, vf_df: DataFrame) -> None: + result = validate_model(vf_df, _VF_TYPE) + assert result.absent_columns == () + + def test_absent_columns_ordered(self, spark: SparkSession) -> None: + # compare_schemas iterates actual fields first, then expected-only fields + # appended in their expected schema order. value precedes sources in + # _VF_SCHEMA, so absent_columns must be exactly ("value", "sources"). + schema_no_value_no_sources = StructType( + [f for f in _VF_SCHEMA.fields if f.name not in {"value", "sources"}] + ) + df = spark.createDataFrame( + [Row(id="1", theme="test", type=_VF_TYPE)], + schema=schema_no_value_no_sources, + ) + result = validate_model(df, _VF_TYPE) + assert result.absent_columns == ("value", "sources") + + def test_absent_columns_deduplicated( + self, spark: SparkSession, _register_nested_type: None + ) -> None: + # A nested struct column with multiple missing sub-fields must produce + # exactly one root entry in absent_columns, not one per sub-field. + # Schema: id + bbox(xmin, xmax) in the expected schema. + # Data: id + bbox(id) -- both xmin and xmax are absent sub-fields, + # so compare_schemas emits bbox.xmin and bbox.xmax as missing; both + # share root "bbox" and must collapse to a single entry. + bbox_partial = StructType([StructField("id", StringType(), True)]) + data_schema = StructType( + [ + StructField("id", StringType(), True), + StructField("bbox", bbox_partial, True), + ] + ) + df = spark.createDataFrame( + [Row(id="1", bbox=Row(id="x"))], + schema=data_schema, + ) + result = validate_model(df, _VF_NESTED_TYPE) + # Two sub-field mismatches (bbox.xmin, bbox.xmax) collapse to one root + assert result.absent_columns == ("bbox",) + + def test_missing_array_nested_field_does_not_raise( + self, spark: SparkSession, _register_array_type: None + ) -> None: + # A field absent from an array element struct yields a mismatch path + # carrying the array step marker (`sources[].confidence`). The absent + # root must be derived by stripping that marker so it matches the + # top-level column (`sources`); the column's checks are then dropped, + # mirroring the top-level graceful-degradation path, rather than + # evaluating an expression that dereferences the absent sub-field. + data_schema = StructType( + [ + StructField("id", StringType(), True), + StructField( + "sources", + ArrayType(StructType([StructField("dataset", StringType(), True)])), + True, + ), + ] + ) + df = spark.createDataFrame( + [Row(id="1", sources=[Row(dataset="osm")])], + schema=data_schema, + ) + result = validate_model(df, _VF_ARRAY_TYPE) + result.row_counts() # forces evaluation; raises if the check is kept + assert result.absent_columns == ("sources",) + assert all("sources" not in c.read_columns for c in result.checks) + + def test_skip_columns_with_map_check_does_not_raise( + self, spark: SparkSession, _register_map_type: None + ) -> None: + # A map key/value check dereferences its map column by name, exactly + # like an array check. Skipping that column must drop the check so + # validation degrades gracefully instead of leaving an unresolvable + # map projection (`map_values_check("license_priority", ...)`) behind. + data_schema = StructType( + [f for f in _VF_MAP_SCHEMA.fields if f.name != "license_priority"] + ) + df = spark.createDataFrame([Row(id="1")], schema=data_schema) + result = validate_model(df, _VF_MAP_TYPE, skip_columns=["license_priority"]) + result.row_counts() # forces evaluation; raises if the map check is kept + assert all("license_priority" not in c.read_columns for c in result.checks) + + def test_suppress_by_model_only_column( + self, spark: SparkSession, _register_model_type: None + ) -> None: + # A column read only by a model-level check is still a valid suppress + # target: suppression is symmetric with absence -- a column droppable + # when absent is suppressible by name. Suppressing it drops the model + # check (and records it as suppressed, not silently absent). + df = spark.createDataFrame( + [Row(id="1", primary_name="p", alt_name="a")], + schema=_VF_MODEL_SCHEMA, + ) + result = validate_model(df, _VF_MODEL_TYPE, suppress=["primary_name"]) + assert all(c.name != "require_any_of" for c in result.checks) + assert any(c.name == "require_any_of" for c in result.suppressed_checks) + + def test_skip_columns_with_model_constraint_does_not_raise( + self, spark: SparkSession, _register_model_type: None + ) -> None: + # Model-level checks read several columns directly. Skipping a column a + # model constraint reads must drop the model check too, so validation + # degrades gracefully instead of leaving an unresolvable column + # reference behind. + data_schema = StructType( + [f for f in _VF_MODEL_SCHEMA.fields if f.name != "primary_name"] + ) + df = spark.createDataFrame( + [Row(id="1", alt_name="x")], + schema=data_schema, + ) + result = validate_model(df, _VF_MODEL_TYPE, skip_columns=["primary_name"]) + result.row_counts() # forces evaluation; raises if the model check is kept + assert all(c.name != "require_any_of" for c in result.checks) diff --git a/packages/overture-schema-system/src/overture/schema/system/case.py b/packages/overture-schema-system/src/overture/schema/system/case.py new file mode 100644 index 000000000..62b3733ae --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/case.py @@ -0,0 +1,26 @@ +"""PascalCase to snake_case conversion.""" + +import re + +__all__ = ["to_snake_case"] + +_ACRONYM_BOUNDARY = re.compile(r"([A-Z]+)([A-Z][a-z])") +_CAMEL_BOUNDARY = re.compile(r"([a-z0-9])([A-Z])") + + +def to_snake_case(name: str) -> str: + """Convert PascalCase to snake_case. + + Handles acronym runs correctly: "HTMLParser" becomes "html_parser", + not "h_t_m_l_parser". + + >>> to_snake_case("HTMLParser") + 'html_parser' + >>> to_snake_case("BuildingPart") + 'building_part' + >>> to_snake_case("simple") + 'simple' + """ + name = _ACRONYM_BOUNDARY.sub(r"\1_\2", name) + name = _CAMEL_BOUNDARY.sub(r"\1_\2", name) + return name.lower() diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py b/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py index ed8af77ad..c894f591e 100644 --- a/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/__init__.py @@ -5,6 +5,12 @@ filter_models, get_registered_model, ) +from .entry_point import ( + entry_point_class_alias, + entry_point_to_path, + resolve_entry_point_key, + split_entry_point, +) from .keys import ModelKey from .types import ModelDict @@ -13,7 +19,11 @@ "ModelKey", "TagSelector", "discover_models", + "entry_point_class_alias", + "entry_point_to_path", "filter_models", "get_registered_model", + "resolve_entry_point_key", + "split_entry_point", "tag", ] diff --git a/packages/overture-schema-system/src/overture/schema/system/discovery/entry_point.py b/packages/overture-schema-system/src/overture/schema/system/discovery/entry_point.py new file mode 100644 index 000000000..270c8addd --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/discovery/entry_point.py @@ -0,0 +1,119 @@ +"""Entry-point string utilities.""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import PurePosixPath + +from ..case import to_snake_case + +__all__ = [ + "entry_point_class_alias", + "entry_point_to_path", + "resolve_entry_point_key", + "split_entry_point", +] + + +def split_entry_point(entry_point_path: str) -> tuple[str, str]: + """Split `"module.path:ClassName"` into dotted module and class name. + + >>> split_entry_point("overture.schema.buildings:Building") + ('overture.schema.buildings', 'Building') + """ + if ":" not in entry_point_path: + msg = f"Expected 'module:Class' format, got {entry_point_path!r}" + raise ValueError(msg) + module, cls = entry_point_path.split(":", 1) + return module, cls + + +def entry_point_to_path(entry_point_path: str) -> tuple[PurePosixPath, str]: + """Translate an entry-point string into a directory path and class name. + + Each dotted component of the module becomes a directory, mirroring + the source package structure. The result is stable regardless of the + set of installed packages. + + Parameters + ---------- + entry_point_path + String in `"module.path:ClassName"` form. + + Returns + ------- + tuple[PurePosixPath, str] + Directory derived from the module path, and the class name. + + Examples + -------- + >>> entry_point_to_path("overture.schema.places:Place") + (PurePosixPath('overture/schema/places'), 'Place') + """ + module, cls = split_entry_point(entry_point_path) + return PurePosixPath(*module.split(".")), cls + + +def entry_point_class_alias(entry_point_path: str) -> str: + """Snake-case class name from an entry-point string. + + The alias is the user-friendly form used to look up entry-point + keys in a registry (e.g. `"place"` resolves + `"overture.schema.places:Place"`). Input without a colon is treated + as a bare class name and snake-cased directly, so the function is + safe to apply to every key in an arbitrary registry mapping. + + Parameters + ---------- + entry_point_path + String in `"module.path:ClassName"` form, or a bare name. + + Examples + -------- + >>> entry_point_class_alias("overture.schema.divisions:DivisionArea") + 'division_area' + """ + cls = entry_point_path.rsplit(":", 1)[-1] + return to_snake_case(cls) + + +def resolve_entry_point_key(name: str, registry: Mapping[str, object]) -> str: + """Resolve a user-supplied name to a canonical entry-point key. + + Tries exact match first, then snake-case class-name alias. Raises + `ValueError` when the alias is ambiguous (matches more than one + registered key) or when the name is unknown. + + Parameters + ---------- + name + User-supplied identifier: an entry-point key or a snake-case + class-name alias. + registry + Mapping whose keys are entry-point strings. + + Returns + ------- + str + The canonical registry key. + + Raises + ------ + ValueError + If `name` matches multiple registry entries via alias, or no + registry entry at all. The message lists the candidates or the + known keys to aid recovery. + """ + if name in registry: + return name + candidates = sorted(k for k in registry if entry_point_class_alias(k) == name) + if len(candidates) == 1: + return candidates[0] + if candidates: + raise ValueError( + f"Entry-point alias {name!r} is ambiguous. " + f"Specify one of: {', '.join(candidates)}" + ) + raise ValueError( + f"Unknown entry-point alias {name!r}. Known: {', '.join(sorted(registry))}" + ) diff --git a/packages/overture-schema-system/src/overture/schema/system/field_constraint/field_constraint.py b/packages/overture-schema-system/src/overture/schema/system/field_constraint/field_constraint.py index d96e1cc4f..5967f391b 100644 --- a/packages/overture-schema-system/src/overture/schema/system/field_constraint/field_constraint.py +++ b/packages/overture-schema-system/src/overture/schema/system/field_constraint/field_constraint.py @@ -8,15 +8,62 @@ one of the peer modules that implements a specific constraint type. """ +import re from abc import ABC, abstractmethod +from collections.abc import Mapping from typing import Any from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler, ValidationInfo from pydantic_core import core_schema +def _normalized(value: object) -> object: + """Reduce a constraint attribute to a hashable, value-stable form. + + A compiled `re.Pattern` carries identity equality -- two patterns built + from the same source compare unequal -- so it reduces to `(pattern, flags)`. + Containers reduce to sorted tuples so equal contents hash equal regardless + of insertion order. The sort by repr is stable because `FieldConstraint` + requires attribute values to be value types (see its docstring), so every + leaf reduces to a value-stable form. + """ + if isinstance(value, re.Pattern): + return (value.pattern, value.flags) + if isinstance(value, Mapping): + return tuple(sorted(((k, _normalized(v)) for k, v in value.items()), key=repr)) + if isinstance(value, (list, tuple)): + return tuple(_normalized(v) for v in value) + if isinstance(value, (set, frozenset)): + return tuple(sorted((_normalized(v) for v in value), key=repr)) + return value + + class FieldConstraint(ABC): - """Base class for field-level constraints.""" + """Base class for field-level constraints. + + Constraints are value objects: two instances of the same concrete type + carrying the same attributes are equal and hash equal, so a set of + constraints deduplicates by rule. Equality keys on the concrete type, so a + fixed-pattern subclass never equals a raw `PatternConstraint` with the same + pattern. + + Subclass attributes participate in equality and hashing, so they must be + value types -- scalars, `re.Pattern`, or containers of these. An attribute + that compares by object identity leaves equality ill-defined. + """ + + def __eq__(self, other: object) -> bool: + if not isinstance(other, FieldConstraint) or type(self) is not type(other): + return NotImplemented + return self._identity() == other._identity() + + def __hash__(self) -> int: + return hash((type(self), self._identity())) + + def _identity(self) -> tuple[tuple[str, object], ...]: + return tuple( + (name, _normalized(value)) for name, value in sorted(vars(self).items()) + ) def validate(self, value: Any, info: ValidationInfo) -> None: # noqa: B027 """Validate the value and raise `ValidationError` if invalid.""" diff --git a/packages/overture-schema-system/src/overture/schema/system/field_path.py b/packages/overture-schema-system/src/overture/schema/system/field_path.py new file mode 100644 index 000000000..7b60fea56 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/field_path.py @@ -0,0 +1,468 @@ +"""Structural representation of a field path through a nested schema. + +A `FieldPath` is one of three variants: + +- `ScalarPath` -- a sequence of `StructSegment` values locating a value + that requires no iteration to reach. +- `ArrayPath` -- a sequence of `StructSegment` and `ArraySegment` values, + with at least one `ArraySegment`, locating a value reached by iterating + one or more arrays. Each `ArraySegment` carries `iter_count`, the number + of `[]` markers on its name in the canonical encoding (multi-depth + segments encode nested-list iteration without an intervening struct, + e.g. `list[list[X]]` parses as a single `ArraySegment` with + `iter_count=2`). +- `MapPath` -- struct segments leading to a map column, a single + `MapSegment` projecting the map to its keys or values, then a struct-only + leaf (possibly empty). Locates a value reached by iterating a + `dict[K, V]`'s keys or values, encoded with a `{key}` / `{value}` marker + on the map column and the leaf appended after it (e.g. `names.common{key}` + for a scalar value, `subs{value}.label` for a field inside a + `dict[K, Model]` value). + +The canonical string form (`str(path)`) round-trips through `parse`. +Code that needs to emit a path into source or labels calls `str(path)` +at the boundary; everything else operates on segments. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import TypeAlias + +__all__ = [ + "ArrayPath", + "ArraySegment", + "FieldPath", + "FieldSegment", + "MapPath", + "MapProjection", + "MapSegment", + "PathSegment", + "ScalarPath", + "StructSegment", + "coerce", + "parse", + "promote_terminal_array", + "promote_terminal_map", +] + + +@dataclass(frozen=True, slots=True) +class StructSegment: + """A struct field navigation step.""" + + name: str + + +@dataclass(frozen=True, slots=True) +class ArraySegment: + """An array column entered with one or more levels of iteration. + + `iter_count` records the number of `[]` markers immediately following + the segment name; values > 1 correspond to nested lists like + `list[list[X]]`. + """ + + name: str + iter_count: int = 1 + + +class MapProjection(Enum): + """Which side of a `dict[K, V]` a `MapSegment` iterates.""" + + KEY = "key" + VALUE = "value" + + +@dataclass(frozen=True, slots=True) +class MapSegment: + """A map column entered by projecting to its keys or values. + + `projection` selects keys or values; the projected side is iterated + like an array, so checks on a `MapSegment` render through the same + element machinery as `ArraySegment`. + """ + + name: str + projection: MapProjection + + +PathSegment: TypeAlias = StructSegment | ArraySegment + + +@dataclass(frozen=True, slots=True) +class ScalarPath: + """Locate a non-iterated value in a row.""" + + segments: tuple[StructSegment, ...] = () + + def append_struct(self, name: str) -> ScalarPath: + return ScalarPath(segments=self.segments + (StructSegment(name=name),)) + + def append_array(self, name: str, iter_count: int = 1) -> ArrayPath: + return ArrayPath( + segments=self.segments + (ArraySegment(name=name, iter_count=iter_count),) + ) + + def __str__(self) -> str: + return ".".join(s.name for s in self.segments) + + +@dataclass(frozen=True, slots=True) +class ArrayPath: + """Locate an iterated value; iteration structure is part of the location. + + Invariant: `segments` contains at least one `ArraySegment`. + """ + + segments: tuple[PathSegment, ...] + + def __post_init__(self) -> None: + if not any(isinstance(s, ArraySegment) for s in self.segments): + raise ValueError("ArrayPath must contain at least one ArraySegment") + + def append_struct(self, name: str) -> ArrayPath: + return ArrayPath(segments=self.segments + (StructSegment(name=name),)) + + def append_array(self, name: str, iter_count: int = 1) -> ArrayPath: + return ArrayPath( + segments=self.segments + (ArraySegment(name=name, iter_count=iter_count),) + ) + + @property + def column_prefix(self) -> ScalarPath: + """Struct segments before the first ArraySegment. + + Returns an empty `ScalarPath(())` when the array is the first + segment. + """ + prefix: list[StructSegment] = [] + for seg in self.segments: + if isinstance(seg, ArraySegment): + break + prefix.append(seg) + return ScalarPath(segments=tuple(prefix)) + + @property + def column_path(self) -> str: + """Dotted name of the outermost array column. + + The struct prefix plus the first ArraySegment's name (unbracketed). + This is what `F.col(...)` or `array_check("...", ...)` consumes. + """ + first_prefix, first_array, _first_iter = self.array_chunks[0] + return ".".join((*first_prefix, first_array)) + + @property + def leaf(self) -> tuple[str, ...]: + """Names of struct segments after the last ArraySegment.""" + last_array = next( + i + for i in range(len(self.segments) - 1, -1, -1) + if isinstance(self.segments[i], ArraySegment) + ) + return tuple(s.name for s in self.segments[last_array + 1 :]) + + @property + def array_chunks( + self, + ) -> tuple[tuple[tuple[str, ...], str, int], ...]: + """One chunk per ArraySegment. + + Each entry is `(prefix_structs, array_name, iter_count)` where + `prefix_structs` is the sequence of struct segment names between + the previous ArraySegment (or the start of the path) and this + ArraySegment. + """ + chunks: list[tuple[tuple[str, ...], str, int]] = [] + prefix: list[str] = [] + for seg in self.segments: + if isinstance(seg, ArraySegment): + chunks.append((tuple(prefix), seg.name, seg.iter_count)) + prefix = [] + else: + prefix.append(seg.name) + return tuple(chunks) + + def element_relative_gate(self, gate: FieldPath) -> tuple[str, ...] | None: + """Path inside this array's element scope that names *gate*. + + Three return states: + + - ``tuple[str, ...]`` (non-empty) -- "reachable with descent": + `gate` enters the same outer array as this path and names a + struct descendant inside its element. The returned segments + name that descendant relative to the element. + - ``()`` -- "reachable, no descent": `gate` is the outer array + itself; the element variable IS the gated value. + - ``None`` -- "not reachable": `gate` does not cross into this + path's element scope (different outer array, scalar gate, + mismatched struct prefix, mismatched boundary `iter_count`, + etc.). Callers must apply the gate at column level instead. + + Raises `NotImplementedError` when `gate` enters the same outer + array but contains a nested `ArraySegment` past the boundary; + the element scope is a struct, so a gate path inside it must be + struct-only. + + Example: `parse("items[].x").element_relative_gate(parse( + "items[].nested")) == ("nested",)`. + """ + column_prefix = self.column_prefix.segments + n_prefix = len(column_prefix) + if not isinstance(gate, ArrayPath): + return None + gate_segs = gate.segments + if len(gate_segs) <= n_prefix: + return None + for i in range(n_prefix): + if not isinstance(gate_segs[i], StructSegment): + return None + if gate_segs[i].name != column_prefix[i].name: + return None + target_boundary = self.segments[n_prefix] + assert isinstance(target_boundary, ArraySegment) + gate_boundary = gate_segs[n_prefix] + if not isinstance(gate_boundary, ArraySegment): + return None + if gate_boundary.name != target_boundary.name: + return None + if gate_boundary.iter_count != target_boundary.iter_count: + return None + inner_segments = gate_segs[n_prefix + 1 :] + for seg in inner_segments: + if not isinstance(seg, StructSegment): + raise NotImplementedError( + f"gate path contains a nested array segment past the " + f"element boundary (gate={gate!r}, self={self!r})" + ) + return tuple(s.name for s in inner_segments) + + @property + def iter_struct_paths(self) -> tuple[tuple[str, ...], ...]: + """Per non-outermost iteration: the struct path that reaches its array. + + For each ArraySegment past the first, emit `(prefix_structs + + array_name)` -- the navigation FROM the previous iteration's + element TO this array. For each `iter_count > 1` on an + ArraySegment, emit `iter_count - 1` additional `()` entries + representing extra iterations inside the same (already-named) + array. + + Returns an empty tuple when the path iterates only once. + """ + paths: list[tuple[str, ...]] = [] + for chunk_idx, (prefix_structs, arr_name, iter_count) in enumerate( + self.array_chunks + ): + if chunk_idx > 0: + paths.append(prefix_structs + (arr_name,)) + for _ in range(iter_count - 1): + paths.append(()) + return tuple(paths) + + def __str__(self) -> str: + return ".".join(_segment_str(s) for s in self.segments) + + +@dataclass(frozen=True, slots=True) +class MapPath: + """Locate a value inside a map's keys or values via one `MapSegment`. + + Invariant: `segments` is a struct prefix, exactly one `MapSegment` + boundary, then a struct-only leaf (possibly empty). The `MapSegment` + iterates the projected keys or values like an array; the leaf navigates + structs inside each projected element, mirroring `ArrayPath.leaf` for a + `list[Model]`. An empty leaf locates the projected scalar itself + (`dict[K, scalar]`); a non-empty leaf locates a field inside a + `dict[K, Model]` value (or key). + + The map must be reachable without array iteration, and the leaf must be + struct-only -- a map nested inside an array element or a container + nested inside a map element is not representable (and + `promote_terminal_map` / `promote_terminal_array` raise rather than + fabricate one). + """ + + segments: tuple[StructSegment | MapSegment, ...] + + def __post_init__(self) -> None: + map_count = sum(isinstance(s, MapSegment) for s in self.segments) + if map_count != 1: + raise ValueError("MapPath must contain exactly one MapSegment") + if not all(isinstance(s, (StructSegment, MapSegment)) for s in self.segments): + raise ValueError("MapPath segments outside the map must be struct segments") + + @property + def _map_index(self) -> int: + return next(i for i, s in enumerate(self.segments) if isinstance(s, MapSegment)) + + @property + def projection(self) -> MapProjection: + seg = self.segments[self._map_index] + assert isinstance(seg, MapSegment) + return seg.projection + + @property + def map_column(self) -> str: + """Dotted name of the map column (struct prefix + map field name). + + This is what `F.col(...)` consumes; the `{key}` / `{value}` marker + and the leaf belong to `str(path)`, not to the column reference. + """ + return ".".join(s.name for s in self.segments[: self._map_index + 1]) + + @property + def leaf(self) -> tuple[str, ...]: + """Names of struct segments after the `MapSegment`. + + Empty for a bare key/value projection; the field path inside each + projected element otherwise. + """ + return tuple(s.name for s in self.segments[self._map_index + 1 :]) + + def append_struct(self, name: str) -> MapPath: + return MapPath(segments=self.segments + (StructSegment(name=name),)) + + def __str__(self) -> str: + base = f"{self.map_column}{{{self.projection.value}}}" + return base + "".join(f".{n}" for n in self.leaf) + + +FieldPath: TypeAlias = ScalarPath | ArrayPath | MapPath + + +# The element type of any `FieldPath.segments`, across all three variants. +# Broader than `PathSegment` (array/scalar paths only): a `MapPath` adds a +# trailing `MapSegment`. Consumers that walk an arbitrary `FieldPath`'s +# segments -- rather than a statically known `ArrayPath` -- annotate with +# this so a `MapSegment` is not a type error. +FieldSegment: TypeAlias = StructSegment | ArraySegment | MapSegment + + +def _segment_str(seg: PathSegment) -> str: + if isinstance(seg, ArraySegment): + return seg.name + "[]" * seg.iter_count + return seg.name + + +def _strip_map_suffix(part: str) -> MapProjection | None: + """Return the `MapProjection` named by a trailing `{key}`/`{value}`, or None.""" + for proj in MapProjection: + if part.endswith(f"{{{proj.value}}}"): + return proj + return None + + +def parse(encoded: str) -> FieldPath: + """Parse a canonical encoded path like `"items[].nested.value"`. + + Trailing `[]` markers on a dotted part produce an `ArraySegment` + with matching `iter_count`; a `{key}`/`{value}` marker produces a + `MapSegment` (and a `MapPath`), with any dotted parts after it forming + the map's struct leaf (e.g. `subs{value}.label`). The empty string + returns the empty `ScalarPath`. Raises `ValueError` when any dotted + part has an empty name (e.g. `".a"`, `"a..b"`, `"[]"`), when more than + one map marker appears, or when an array marker combines with a map + projection (`dict[K, list[V]]` is not representable as a `MapPath`). + """ + if not encoded: + return ScalarPath() + segments: list[StructSegment | ArraySegment | MapSegment] = [] + struct_segments: list[StructSegment] = [] + has_array = False + map_seen = False + parts = encoded.split(".") + for part in parts: + projection = _strip_map_suffix(part) + if projection is not None: + if map_seen: + raise ValueError(f"FieldPath has multiple map markers in {encoded!r}") + part = part[: -(len(projection.value) + 2)] + depth = 0 + while part.endswith("[]"): + part = part[:-2] + depth += 1 + if not part: + raise ValueError(f"FieldPath part has empty name in {encoded!r}") + if projection is not None: + if depth > 0: + raise ValueError( + f"map projection marker cannot follow array markers in {encoded!r}" + ) + map_seen = True + segments.append(MapSegment(name=part, projection=projection)) + elif depth > 0: + has_array = True + segments.append(ArraySegment(name=part, iter_count=depth)) + else: + struct = StructSegment(name=part) + segments.append(struct) + struct_segments.append(struct) + if map_seen: + if has_array: + raise ValueError( + f"map projection cannot combine with array markers in {encoded!r}" + ) + return MapPath(segments=tuple(segments)) # type: ignore[arg-type] + if has_array: + # No MapSegment reached this branch (map_seen is False), so the + # tuple holds only Struct/Array segments. + return ArrayPath(segments=tuple(segments)) # type: ignore[arg-type] + return ScalarPath(segments=tuple(struct_segments)) + + +def coerce(value: FieldPath | str) -> FieldPath: + """Return *value* as a `FieldPath`, parsing it from string if needed.""" + if isinstance(value, str): + return parse(value) + return value + + +def promote_terminal_array(path: FieldPath) -> ArrayPath: + """Promote *path*'s terminal segment to an iterated `ArraySegment`. + + A `StructSegment` terminal is *replaced* with `ArraySegment(name, + iter_count=1)`; an `ArraySegment` terminal has its `iter_count` + incremented. This is how a walker records entering a `list[...]` + layer on the field it is already pointing at -- unlike `append_array`, + which adds a new segment for a fresh nested array. Repeated calls + build the multi-iteration terminal of a `list[list[X]]` field. + + Raises `ValueError` on an empty path: there is no terminal segment + to promote. Raises `NotImplementedError` for a `MapPath`: a list nested + inside a map element has no representable path, so the gap stays loud. + """ + if not path.segments: + raise ValueError("cannot promote the terminal of an empty path") + if isinstance(path, MapPath): + raise NotImplementedError("list nested inside a map element is not supported") + *prefix, last = path.segments + if isinstance(last, ArraySegment): + promoted = ArraySegment(name=last.name, iter_count=last.iter_count + 1) + else: + promoted = ArraySegment(name=last.name, iter_count=1) + return ArrayPath(segments=(*prefix, promoted)) + + +def promote_terminal_map(path: FieldPath, projection: MapProjection) -> MapPath: + """Promote *path*'s terminal struct segment to a `MapSegment`. + + Records a walker entering a `dict[K, V]` layer on the field it already + points at, projecting to keys or values. Raises `ValueError` on an + empty path and `NotImplementedError` when the map is reached through + array iteration or already projects another map -- a map nested inside + an array element or another map element has no schema field today and + no representable `MapPath`, so the gap stays loud. + """ + if not path.segments: + raise ValueError("cannot promote the terminal of an empty path") + if isinstance(path, ArrayPath): + raise NotImplementedError("map nested under a list layer is not supported") + if isinstance(path, MapPath): + raise NotImplementedError("map nested inside a map element is not supported") + *prefix, last = path.segments + return MapPath( + segments=(*prefix, MapSegment(name=last.name, projection=projection)) # type: ignore[arg-type] + ) diff --git a/packages/overture-schema-system/tests/field_constraint/test_constraint_equality.py b/packages/overture-schema-system/tests/field_constraint/test_constraint_equality.py new file mode 100644 index 000000000..c723df812 --- /dev/null +++ b/packages/overture-schema-system/tests/field_constraint/test_constraint_equality.py @@ -0,0 +1,109 @@ +"""Value-equality semantics for `FieldConstraint` subclasses. + +Two constraints of the same concrete type with the same attributes are equal +and hash equal, so a set of constraints deduplicates by rule rather than by +object identity. Equality keys on the concrete type, so a subclass with a fixed +pattern never equals a raw `PatternConstraint` carrying the same pattern. +""" + +import re +from typing import Any + +from pydantic import GetCoreSchemaHandler +from pydantic_core import core_schema + +from overture.schema.system.field_constraint import ( + CountryCodeAlpha2Constraint, + FieldConstraint, + HexColorConstraint, + JsonPointerConstraint, + PatternConstraint, + UniqueItemsConstraint, +) + + +class TestMarkerConstraintEquality: + def test_equal_instances_compare_and_hash_equal(self) -> None: + a, b = UniqueItemsConstraint(), UniqueItemsConstraint() + assert a == b + assert hash(a) == hash(b) + assert len({a, b}) == 1 + + def test_distinct_marker_classes_unequal(self) -> None: + assert UniqueItemsConstraint() != JsonPointerConstraint() + + +class TestParametricConstraintEquality: + def test_fixed_pattern_subclass_instances_equal(self) -> None: + a, b = CountryCodeAlpha2Constraint(), CountryCodeAlpha2Constraint() + assert a == b + assert hash(a) == hash(b) + assert len({a, b}) == 1 + + def test_distinct_pattern_subclasses_unequal(self) -> None: + assert CountryCodeAlpha2Constraint() != HexColorConstraint() + + def test_equal_raw_patterns_collapse(self) -> None: + a = PatternConstraint(r"^[a-z]+$", "err") + b = PatternConstraint(r"^[a-z]+$", "err") + assert a == b + assert hash(a) == hash(b) + + def test_pattern_flags_distinguish(self) -> None: + a = PatternConstraint(r"^[a-z]+$", "err") + b = PatternConstraint(r"^[a-z]+$", "err", re.IGNORECASE) + assert a != b + + def test_subclass_not_equal_to_base_with_same_state(self) -> None: + """A fixed-pattern subclass is a distinct rule from a raw equivalent.""" + country = CountryCodeAlpha2Constraint() + raw = PatternConstraint( + country.pattern.pattern, + country.error_message, + description=country.description, + min_length=country.min_length, + max_length=country.max_length, + ) + assert country != raw + + +class _ListConstraint(FieldConstraint): + """Test-only constraint with a list-valued attribute.""" + + def __init__(self, items: list[str]) -> None: + self.items = list(items) + + def __get_pydantic_core_schema__( + self, source: type[Any], handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return handler(source) + + +class _DictConstraint(FieldConstraint): + """Test-only constraint with a dict-valued attribute.""" + + def __init__(self, mapping: dict[str, int]) -> None: + self.mapping = dict(mapping) + + def __get_pydantic_core_schema__( + self, source: type[Any], handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return handler(source) + + +class TestContainerValuedAttributes: + """A future constraint with a container attribute stays a hashable value.""" + + def test_equal_list_attr_instances_collapse(self) -> None: + a, b = _ListConstraint(["a", "b"]), _ListConstraint(["a", "b"]) + assert a == b + assert len({a, b}) == 1 + + def test_distinct_list_attr_instances_unequal(self) -> None: + assert _ListConstraint(["a", "b"]) != _ListConstraint(["a", "c"]) + + def test_equal_dict_attr_instances_collapse(self) -> None: + a = _DictConstraint({"x": 1, "y": 2}) + b = _DictConstraint({"y": 2, "x": 1}) + assert a == b + assert len({a, b}) == 1 diff --git a/packages/overture-schema-system/tests/field_constraint/test_string_constraints.py b/packages/overture-schema-system/tests/field_constraint/test_string_constraints.py index 14a1ebae1..0ba1be6ce 100644 --- a/packages/overture-schema-system/tests/field_constraint/test_string_constraints.py +++ b/packages/overture-schema-system/tests/field_constraint/test_string_constraints.py @@ -1,3 +1,4 @@ +import re from typing import Annotated import pytest @@ -210,7 +211,6 @@ class TestModel(BaseModel): def test_stripped_constraint_json_schema_pattern(self) -> None: """StrippedConstraint's JSON schema pattern accepts empty string and rejects leading/trailing whitespace.""" - import re class TestModel(BaseModel): text: Annotated[str, StrippedConstraint()] diff --git a/packages/overture-schema-codegen/tests/test_naming.py b/packages/overture-schema-system/tests/test_case.py similarity index 67% rename from packages/overture-schema-codegen/tests/test_naming.py rename to packages/overture-schema-system/tests/test_case.py index 77e4d5773..21cddcb5a 100644 --- a/packages/overture-schema-codegen/tests/test_naming.py +++ b/packages/overture-schema-system/tests/test_case.py @@ -1,7 +1,8 @@ """Tests for PascalCase to snake_case conversion.""" import pytest -from overture.schema.codegen.extraction.case_conversion import to_snake_case + +from overture.schema.system.case import to_snake_case class TestToSnakeCase: @@ -14,10 +15,11 @@ class TestToSnakeCase: ("BuildingPart", "building_part"), ("RoadSegment", "road_segment"), ("Place", "place"), - ("simple", "simple"), # Already lowercase - ("HTTPServer", "http_server"), # Consecutive caps + ("simple", "simple"), + ("HTTPServer", "http_server"), + ("HTMLParser", "html_parser"), ], ) def test_converts_pascal_to_snake(self, input_name: str, expected: str) -> None: - """PascalCase names should convert to snake_case.""" + """PascalCase names convert to snake_case; acronyms collapse.""" assert to_snake_case(input_name) == expected diff --git a/packages/overture-schema-system/tests/test_discovery_entry_point.py b/packages/overture-schema-system/tests/test_discovery_entry_point.py new file mode 100644 index 000000000..3f8c766af --- /dev/null +++ b/packages/overture-schema-system/tests/test_discovery_entry_point.py @@ -0,0 +1,97 @@ +"""Tests for entry-point string utilities.""" + +from pathlib import PurePosixPath + +import pytest + +from overture.schema.system.discovery.entry_point import ( + entry_point_class_alias, + entry_point_to_path, + resolve_entry_point_key, +) + + +class TestEntryPointToPath: + def test_typical_overture_entry_point(self) -> None: + path, cls = entry_point_to_path("overture.schema.places:Place") + assert path == PurePosixPath("overture/schema/places") + assert cls == "Place" + + def test_single_segment_module(self) -> None: + path, cls = entry_point_to_path("myschema:Foo") + assert path == PurePosixPath("myschema") + assert cls == "Foo" + + def test_deeply_nested_module(self) -> None: + path, cls = entry_point_to_path("a.b.c.d.e:Thing") + assert path == PurePosixPath("a/b/c/d/e") + assert cls == "Thing" + + def test_missing_colon_raises(self) -> None: + with pytest.raises(ValueError, match="module:Class"): + entry_point_to_path("overture.schema.places.Place") + + def test_class_name_with_dot_kept(self) -> None: + # Class name after the colon is taken verbatim — Python class + # names can't contain dots, but we don't validate. + path, cls = entry_point_to_path("a.b:Outer.Inner") + assert path == PurePosixPath("a/b") + assert cls == "Outer.Inner" + + +class TestEntryPointClassAlias: + def test_returns_snake_case_class_name(self) -> None: + assert entry_point_class_alias("overture.schema.places:Place") == "place" + + def test_handles_pascal_case_class(self) -> None: + assert ( + entry_point_class_alias("overture.schema.buildings:BuildingPart") + == "building_part" + ) + + def test_handles_acronyms(self) -> None: + assert ( + entry_point_class_alias("overture.schema.places:HTMLParser") + == "html_parser" + ) + + def test_bare_name_is_snake_cased(self) -> None: + # Tolerant of registry keys that aren't entry-point-formatted — + # the snake-case form of the whole string is returned. + assert entry_point_class_alias("BareName") == "bare_name" + + +class TestResolveEntryPointKey: + def test_exact_match(self) -> None: + registry = {"overture.schema.places:Place": object()} + assert ( + resolve_entry_point_key("overture.schema.places:Place", registry) + == "overture.schema.places:Place" + ) + + def test_snake_case_alias_match(self) -> None: + registry = {"overture.schema.places:Place": object()} + assert ( + resolve_entry_point_key("place", registry) == "overture.schema.places:Place" + ) + + def test_ambiguous_lists_candidates(self) -> None: + registry = { + "overture.schema.places:Place": object(), + "annex.schema.places:Place": object(), + } + with pytest.raises(ValueError, match="ambiguous"): + resolve_entry_point_key("place", registry) + + def test_unknown_lists_known(self) -> None: + registry = {"overture.schema.places:Place": object()} + with pytest.raises(ValueError, match="Unknown"): + resolve_entry_point_key("zzz", registry) + + def test_acronym_class_name_resolves(self) -> None: + registry = { + "ns.a:HTMLParser": object(), + "ns.b:HTMLParser": object(), + } + with pytest.raises(ValueError, match=r"ns\.a:HTMLParser"): + resolve_entry_point_key("html_parser", registry) diff --git a/packages/overture-schema-system/tests/test_field_path.py b/packages/overture-schema-system/tests/test_field_path.py new file mode 100644 index 000000000..4c10c5e3f --- /dev/null +++ b/packages/overture-schema-system/tests/test_field_path.py @@ -0,0 +1,615 @@ +"""Tests for FieldPath, the structural path type for nested schemas.""" + +from __future__ import annotations + +import re + +import pytest + +from overture.schema.system.field_path import ( + ArrayPath, + ArraySegment, + MapPath, + MapProjection, + MapSegment, + ScalarPath, + StructSegment, + coerce, + parse, + promote_terminal_array, + promote_terminal_map, +) + + +class TestParseAndRoundTrip: + def test_empty_path_parses_to_empty_scalar(self) -> None: + assert parse("") == ScalarPath(segments=()) + + def test_single_segment(self) -> None: + path = parse("name") + assert path == ScalarPath(segments=(StructSegment(name="name"),)) + + def test_dotted_path(self) -> None: + path = parse("bbox.xmin") + assert path == ScalarPath( + segments=(StructSegment(name="bbox"), StructSegment(name="xmin")) + ) + + def test_array_segment(self) -> None: + path = parse("items[]") + assert path == ArrayPath(segments=(ArraySegment(name="items", iter_count=1),)) + + def test_array_with_nested_field(self) -> None: + path = parse("items[].value") + assert path == ArrayPath( + segments=( + ArraySegment(name="items", iter_count=1), + StructSegment(name="value"), + ) + ) + + def test_nested_list_depth(self) -> None: + path = parse("hierarchies[][]") + assert path == ArrayPath( + segments=(ArraySegment(name="hierarchies", iter_count=2),) + ) + + def test_nested_list_with_leaf(self) -> None: + path = parse("hierarchies[][].value") + assert path == ArrayPath( + segments=( + ArraySegment(name="hierarchies", iter_count=2), + StructSegment(name="value"), + ) + ) + + def test_complex_path(self) -> None: + path = parse("speed_limits[].when.vehicle[].dimension") + assert path == ArrayPath( + segments=( + ArraySegment(name="speed_limits", iter_count=1), + StructSegment(name="when"), + ArraySegment(name="vehicle", iter_count=1), + StructSegment(name="dimension"), + ) + ) + + @pytest.mark.parametrize( + "encoded", + [ + "", + "name", + "bbox.xmin", + "items[]", + "items[].value", + "hierarchies[][]", + "hierarchies[][].value", + "speed_limits[].when.vehicle[].dimension", + "tags_min_length", + ], + ) + def test_str_round_trip(self, encoded: str) -> None: + assert str(parse(encoded)) == encoded + + +class TestScalarVsArrayPartition: + def test_no_array_returns_scalar_path(self) -> None: + assert isinstance(parse("a.b.c"), ScalarPath) + + def test_with_array_returns_array_path(self) -> None: + assert isinstance(parse("a.b[].c"), ArrayPath) + + def test_empty_is_scalar(self) -> None: + assert isinstance(parse(""), ScalarPath) + + +class TestStr: + def test_empty_renders_as_empty(self) -> None: + assert str(ScalarPath()) == "" + + def test_scalar_path_renders_dotted(self) -> None: + path = ScalarPath( + segments=(StructSegment(name="bbox"), StructSegment(name="xmin")) + ) + assert str(path) == "bbox.xmin" + + def test_array_path_renders_with_brackets(self) -> None: + path = ArrayPath( + segments=( + ArraySegment(name="speed_limits", iter_count=1), + StructSegment(name="when"), + ) + ) + assert str(path) == "speed_limits[].when" + + def test_array_path_renders_multi_depth(self) -> None: + path = ArrayPath(segments=(ArraySegment(name="hierarchies", iter_count=2),)) + assert str(path) == "hierarchies[][]" + + +class TestAppendStruct: + def test_scalar_append_struct_returns_scalar(self) -> None: + path = ScalarPath().append_struct("name") + assert path == parse("name") + assert isinstance(path, ScalarPath) + + def test_scalar_chain_struct(self) -> None: + path = ScalarPath().append_struct("bbox").append_struct("xmin") + assert path == parse("bbox.xmin") + + def test_array_append_struct_returns_array(self) -> None: + path = parse("items[]") + assert isinstance(path, ArrayPath) + result = path.append_struct("value") + assert result == parse("items[].value") + assert isinstance(result, ArrayPath) + + +class TestAppendArray: + def test_scalar_append_array_returns_array_path(self) -> None: + path = ScalarPath().append_array("items") + assert path == parse("items[]") + assert isinstance(path, ArrayPath) + + def test_scalar_append_array_after_struct(self) -> None: + path = ScalarPath().append_struct("outer").append_array("items") + assert path == parse("outer.items[]") + + def test_scalar_append_array_multi_depth(self) -> None: + path = ScalarPath().append_array("hierarchies", iter_count=2) + assert path == parse("hierarchies[][]") + + def test_array_append_array(self) -> None: + path = parse("outer[]") + assert isinstance(path, ArrayPath) + result = path.append_array("inner") + assert result == parse("outer[].inner[]") + + +class TestPromoteTerminalArray: + def test_scalar_struct_terminal_becomes_array(self) -> None: + assert promote_terminal_array(parse("tags")) == parse("tags[]") + + def test_struct_prefix_is_preserved(self) -> None: + assert promote_terminal_array(parse("outer.tags")) == parse("outer.tags[]") + + def test_struct_terminal_inside_array_path(self) -> None: + assert promote_terminal_array(parse("items[].tags")) == parse("items[].tags[]") + + def test_array_terminal_increments_iter_count(self) -> None: + assert promote_terminal_array(parse("tags[]")) == parse("tags[][]") + + def test_consecutive_promotions_stack(self) -> None: + assert promote_terminal_array(promote_terminal_array(parse("grid"))) == parse( + "grid[][]" + ) + + def test_array_terminal_inside_array_path(self) -> None: + assert promote_terminal_array(parse("items[].grid[]")) == parse( + "items[].grid[][]" + ) + + def test_empty_path_raises(self) -> None: + with pytest.raises(ValueError, match="empty path"): + promote_terminal_array(ScalarPath()) + + def test_map_path_raises(self) -> None: + with pytest.raises(NotImplementedError, match="map"): + promote_terminal_array(parse("subs{value}.inner")) + + +class TestColumnPrefix: + def test_array_at_start_has_empty_prefix(self) -> None: + path = parse("items[].value") + assert isinstance(path, ArrayPath) + assert path.column_prefix == ScalarPath(()) + + def test_struct_prefix_before_array(self) -> None: + path = parse("parent.items[].value") + assert isinstance(path, ArrayPath) + assert path.column_prefix == parse("parent") + + def test_dotted_struct_prefix(self) -> None: + path = parse("a.b.c[].d") + assert isinstance(path, ArrayPath) + assert path.column_prefix == parse("a.b") + + +class TestLeaf: + def test_no_leaf_after_array(self) -> None: + path = parse("items[]") + assert isinstance(path, ArrayPath) + assert path.leaf == () + + def test_single_struct_leaf(self) -> None: + path = parse("items[].value") + assert isinstance(path, ArrayPath) + assert path.leaf == ("value",) + + def test_nested_struct_leaf(self) -> None: + path = parse("items[].nested.value") + assert isinstance(path, ArrayPath) + assert path.leaf == ("nested", "value") + + def test_uses_last_array(self) -> None: + path = parse("speed_limits[].when.vehicle[].dimension") + assert isinstance(path, ArrayPath) + assert path.leaf == ("dimension",) + + +class TestArrayChunks: + def test_single_top_level_array(self) -> None: + path = parse("items[]") + assert isinstance(path, ArrayPath) + assert path.array_chunks == (((), "items", 1),) + + def test_single_array_with_struct_prefix(self) -> None: + path = parse("parent.items[].value") + assert isinstance(path, ArrayPath) + assert path.array_chunks == ((("parent",), "items", 1),) + + def test_nested_arrays(self) -> None: + path = parse("speed_limits[].when.vehicle[].dimension") + assert isinstance(path, ArrayPath) + assert path.array_chunks == ( + ((), "speed_limits", 1), + (("when",), "vehicle", 1), + ) + + def test_multi_depth_array(self) -> None: + path = parse("hierarchies[][].value") + assert isinstance(path, ArrayPath) + assert path.array_chunks == (((), "hierarchies", 2),) + + +class TestIterStructPaths: + def test_single_iteration_is_empty(self) -> None: + path = parse("items[].value") + assert isinstance(path, ArrayPath) + assert path.iter_struct_paths == () + + def test_nested_arrays_emit_navigation_path(self) -> None: + path = parse("speed_limits[].when.vehicle[].dimension") + assert isinstance(path, ArrayPath) + assert path.iter_struct_paths == (("when", "vehicle"),) + + def test_multi_depth_array_expands_extra_iterations(self) -> None: + path = parse("hierarchies[][].value") + assert isinstance(path, ArrayPath) + assert path.iter_struct_paths == ((),) + + def test_multi_depth_inner_array_combines_navigation_and_expansion(self) -> None: + path = parse("rules[].tags[][].value") + assert isinstance(path, ArrayPath) + assert path.iter_struct_paths == (("tags",), ()) + + +class TestElementRelativeGate: + def test_gate_inside_same_outer_array(self) -> None: + target = parse("items[].value") + gate = parse("items[].nested") + assert isinstance(target, ArrayPath) + assert target.element_relative_gate(gate) == ("nested",) + + def test_gate_at_outer_array_root_returns_empty(self) -> None: + target = parse("items[].value") + gate = parse("items[]") + assert isinstance(target, ArrayPath) + assert target.element_relative_gate(gate) == () + + def test_gate_with_dotted_struct_inside_element(self) -> None: + target = parse("items[].value") + gate = parse("items[].a.b") + assert isinstance(target, ArrayPath) + assert target.element_relative_gate(gate) == ("a", "b") + + def test_scalar_gate_returns_none(self) -> None: + target = parse("items[].value") + gate = parse("other") + assert isinstance(target, ArrayPath) + assert target.element_relative_gate(gate) is None + + def test_different_outer_array_returns_none(self) -> None: + target = parse("items[].value") + gate = parse("other[].x") + assert isinstance(target, ArrayPath) + assert target.element_relative_gate(gate) is None + + def test_struct_prefix_must_match(self) -> None: + target = parse("parent.items[].value") + gate = parse("items[].x") + assert isinstance(target, ArrayPath) + assert target.element_relative_gate(gate) is None + + def test_matching_struct_prefix(self) -> None: + target = parse("parent.items[].value") + gate = parse("parent.items[].x") + assert isinstance(target, ArrayPath) + assert target.element_relative_gate(gate) == ("x",) + + def test_inner_array_segment_raises(self) -> None: + target = parse("items[].value") + gate = parse("items[].nested[]") + assert isinstance(target, ArrayPath) + with pytest.raises(NotImplementedError, match="nested array segment"): + target.element_relative_gate(gate) + + def test_mismatched_iter_count_returns_none(self) -> None: + # target iterates items[] (iter_count=1); gate enters items[][] (iter_count=2) + # -- same name, different iteration depth -- not the same element scope + target = parse("items[].value") + gate = parse("items[][].nested") + assert isinstance(target, ArrayPath) + assert target.element_relative_gate(gate) is None + + def test_matching_iter_count_still_returns_element_relative_tuple(self) -> None: + # regression: matching iter_count must remain reachable after the fix + target = parse("items[][].value") + gate = parse("items[][].nested") + assert isinstance(target, ArrayPath) + assert target.element_relative_gate(gate) == ("nested",) + + +class TestArrayPathInvariant: + def test_rejects_segments_without_array(self) -> None: + with pytest.raises(ValueError, match="at least one ArraySegment"): + ArrayPath(segments=(StructSegment(name="a"),)) + + +class TestEqualityAndHashing: + def test_paths_with_same_segments_are_equal(self) -> None: + assert parse("items[].value") == parse("items[].value") + + def test_different_paths_unequal(self) -> None: + assert parse("items[].value") != parse("items[].other") + + def test_scalar_array_unequal(self) -> None: + assert parse("items") != parse("items[]") + + def test_hashable(self) -> None: + s = {parse("a.b"), parse("a.b"), parse("c")} + assert len(s) == 2 + + def test_string_is_not_equal_to_path(self) -> None: + assert parse("items[].value") != "items[].value" + + +class TestCoerce: + def test_passes_through_scalar(self) -> None: + path = parse("a.b") + assert coerce(path) is path + + def test_passes_through_array(self) -> None: + path = parse("items[].value") + assert coerce(path) is path + + def test_parses_string(self) -> None: + assert coerce("items[].value") == parse("items[].value") + + +class TestMapPath: + def test_str_top_level_key(self) -> None: + path = MapPath( + segments=(MapSegment(name="tags", projection=MapProjection.KEY),) + ) + assert str(path) == "tags{key}" + + def test_str_top_level_value(self) -> None: + path = MapPath( + segments=(MapSegment(name="tags", projection=MapProjection.VALUE),) + ) + assert str(path) == "tags{value}" + + def test_str_nested_under_struct(self) -> None: + path = MapPath( + segments=( + StructSegment(name="names"), + MapSegment(name="common", projection=MapProjection.KEY), + ) + ) + assert str(path) == "names.common{key}" + + def test_projection_property(self) -> None: + path = MapPath( + segments=(MapSegment(name="tags", projection=MapProjection.VALUE),) + ) + assert path.projection is MapProjection.VALUE + + def test_map_column_top_level(self) -> None: + path = MapPath( + segments=(MapSegment(name="tags", projection=MapProjection.KEY),) + ) + assert path.map_column == "tags" + + def test_map_column_nested(self) -> None: + path = MapPath( + segments=( + StructSegment(name="names"), + MapSegment(name="common", projection=MapProjection.VALUE), + ) + ) + assert path.map_column == "names.common" + + def test_must_contain_a_map_segment(self) -> None: + with pytest.raises(ValueError, match="MapSegment"): + MapPath(segments=(StructSegment(name="names"),)) + + def test_rejects_array_segment_before_map(self) -> None: + with pytest.raises(ValueError, match="struct"): + MapPath( + segments=( # type: ignore[arg-type] # invalid by design: runtime guard under test + ArraySegment(name="items"), + MapSegment(name="tags", projection=MapProjection.KEY), + ) + ) + + def test_rejects_two_map_segments(self) -> None: + with pytest.raises(ValueError, match="MapSegment"): + MapPath( + segments=( + MapSegment(name="a", projection=MapProjection.KEY), + MapSegment(name="b", projection=MapProjection.VALUE), + ) + ) + + @pytest.mark.parametrize( + "encoded", + ["tags{key}", "tags{value}", "names.common{key}", "names.common{value}"], + ) + def test_str_round_trip(self, encoded: str) -> None: + assert str(parse(encoded)) == encoded + + def test_parse_returns_map_path(self) -> None: + assert isinstance(parse("names.common{key}"), MapPath) + + def test_parse_key(self) -> None: + assert parse("tags{key}") == MapPath( + segments=(MapSegment(name="tags", projection=MapProjection.KEY),) + ) + + def test_parse_nested_value(self) -> None: + assert parse("names.common{value}") == MapPath( + segments=( + StructSegment(name="names"), + MapSegment(name="common", projection=MapProjection.VALUE), + ) + ) + + +class TestMapPathLeaf: + """A `MapPath` may carry struct segments after the `MapSegment`. + + These name a value inside a `dict[K, Model]`'s value (or key) struct, + mirroring `ArrayPath.leaf` for `list[Model]`. The `MapSegment` is the + iteration boundary; the leaf is the struct navigation inside each + projected element. + """ + + def test_leaf_empty_for_bare_projection(self) -> None: + path = MapPath( + segments=(MapSegment(name="subs", projection=MapProjection.VALUE),) + ) + assert path.leaf == () + + def test_leaf_names_struct_segments_after_map(self) -> None: + path = MapPath( + segments=( + MapSegment(name="subs", projection=MapProjection.VALUE), + StructSegment(name="label"), + ) + ) + assert path.leaf == ("label",) + + def test_leaf_spans_nested_struct_navigation(self) -> None: + path = MapPath( + segments=( + MapSegment(name="subs", projection=MapProjection.VALUE), + StructSegment(name="inner"), + StructSegment(name="label"), + ) + ) + assert path.leaf == ("inner", "label") + + def test_map_column_excludes_leaf(self) -> None: + path = MapPath( + segments=( + StructSegment(name="names"), + MapSegment(name="common", projection=MapProjection.VALUE), + StructSegment(name="label"), + ) + ) + assert path.map_column == "names.common" + + def test_projection_found_with_leaf_present(self) -> None: + path = MapPath( + segments=( + MapSegment(name="subs", projection=MapProjection.KEY), + StructSegment(name="label"), + ) + ) + assert path.projection is MapProjection.KEY + + def test_str_appends_leaf_after_marker(self) -> None: + path = MapPath( + segments=( + MapSegment(name="subs", projection=MapProjection.VALUE), + StructSegment(name="label"), + ) + ) + assert str(path) == "subs{value}.label" + + def test_append_struct_extends_leaf(self) -> None: + path = MapPath( + segments=(MapSegment(name="subs", projection=MapProjection.VALUE),) + ) + extended = path.append_struct("label") + assert extended == MapPath( + segments=( + MapSegment(name="subs", projection=MapProjection.VALUE), + StructSegment(name="label"), + ) + ) + + def test_rejects_array_segment_in_leaf(self) -> None: + with pytest.raises(ValueError, match="struct"): + MapPath( + segments=( # type: ignore[arg-type] # invalid by design: runtime guard under test + MapSegment(name="subs", projection=MapProjection.VALUE), + ArraySegment(name="items"), + ) + ) + + @pytest.mark.parametrize( + "encoded", + ["subs{value}.label", "names.common{key}.tag", "subs{value}.inner.label"], + ) + def test_str_round_trip_with_leaf(self, encoded: str) -> None: + assert str(parse(encoded)) == encoded + + def test_parse_value_with_leaf(self) -> None: + assert parse("subs{value}.label") == MapPath( + segments=( + MapSegment(name="subs", projection=MapProjection.VALUE), + StructSegment(name="label"), + ) + ) + + def test_parse_rejects_array_marker_in_leaf(self) -> None: + with pytest.raises(ValueError, match="map projection"): + parse("subs{value}.items[]") + + +class TestPromoteTerminalMap: + def test_top_level_struct_becomes_map_key(self) -> None: + assert promote_terminal_map(parse("tags"), MapProjection.KEY) == parse( + "tags{key}" + ) + + def test_struct_prefix_preserved_for_value(self) -> None: + assert promote_terminal_map( + parse("names.common"), MapProjection.VALUE + ) == parse("names.common{value}") + + def test_empty_path_raises(self) -> None: + with pytest.raises(ValueError, match="empty path"): + promote_terminal_map(ScalarPath(), MapProjection.KEY) + + def test_array_path_raises(self) -> None: + with pytest.raises(NotImplementedError, match="list"): + promote_terminal_map(parse("items[].tags"), MapProjection.KEY) + + def test_map_path_raises(self) -> None: + with pytest.raises(NotImplementedError, match="map"): + promote_terminal_map(parse("subs{value}.inner"), MapProjection.VALUE) + + +class TestParseRejectsEmptyParts: + @pytest.mark.parametrize("encoded", [".a", "a..b", "[]", "a.[]", ".[]"]) + def test_raises_value_error_on_empty_part(self, encoded: str) -> None: + with pytest.raises(ValueError, match="empty name"): + parse(encoded) + + @pytest.mark.parametrize("encoded", [".a", "a..b", "[]"]) + def test_error_includes_input_string(self, encoded: str) -> None: + with pytest.raises(ValueError, match=re.escape(repr(encoded))): + parse(encoded) diff --git a/packages/overture-schema-transportation-theme/pyproject.toml b/packages/overture-schema-transportation-theme/pyproject.toml index 547b54401..7824ec665 100644 --- a/packages/overture-schema-transportation-theme/pyproject.toml +++ b/packages/overture-schema-transportation-theme/pyproject.toml @@ -1,6 +1,6 @@ [project] maintainers = [ - {name = "Overture Maps Schema Working Group"}, + { name = "Overture Maps Schema Working Group" }, ] dependencies = [ "overture-schema-common", @@ -41,62 +41,158 @@ segment = "overture.schema.transportation:Segment" [project.entry-points.pytest11] overture_baselines = "overture.schema.system.testing.plugin" +# Connector: Vermooten Street junction, Pretoria, South Africa (2026-02-18.0) [[examples.Connector]] -id = "39542bee-230f-4b91-b7e5-a9b58e0c59b1" -geometry = "POINT (-176.5472979 -43.9679472)" -version = 1 +geometry = "POINT (30.048398 -25.708697)" +id = "73a46c48-dc5a-4162-b9c8-1643298784c3" theme = "transportation" type = "connector" +version = 1 [examples.Connector.bbox] -xmin = -176.54730224609375 -xmax = -176.54727172851562 -ymin = -43.96794891357422 -ymax = -43.96794128417969 +xmax = 30.04840087890625 +xmin = 30.048397064208984 +ymax = -25.708696365356445 +ymin = -25.70870018005371 [[examples.Connector.sources]] -property = "" dataset = "OpenStreetMap" +license = "ODbL-1.0" +property = "" +record_id = "n252436807@6" +update_time = "2025-01-06T20:44:06Z" +# Road segment: Vermooten Street / R33, Pretoria, South Africa (2026-02-18.0) +# Populates access_restrictions with when.vehicle, speed_limits with +# when.heading, routes with ref, road_surface, and names. [[examples.Segment]] -id = "1bc62f3b-08b5-42b8-89fe-36f685f60455" -geometry = "LINESTRING (-176.5636191 -43.954404, -176.5643637 -43.9538145, -176.5647264 -43.9535274, -176.5649947 -43.953251)" -version = 1 +class = "primary" +geometry = "LINESTRING (30.048398 -25.708697, 30.0485458 -25.708892, 30.0487074 -25.7090728, 30.0488875 -25.709252, 30.049138 -25.7094697, 30.0493666 -25.7096603, 30.0497209 -25.7099369, 30.0509508 -25.710904, 30.0511567 -25.7110786, 30.0515268 -25.7113855, 30.0518399 -25.711661, 30.052143 -25.7119631, 30.0523513 -25.7121619, 30.0526875 -25.7124827, 30.0531992 -25.7129799, 30.0535874 -25.7133575)" +id = "621e0a00-9466-4c3f-bb4a-64a83cd7a934" subtype = "road" -class = "residential" theme = "transportation" type = "segment" +version = 4 [examples.Segment.bbox] -xmin = -176.5650177001953 -xmax = -176.56361389160156 -ymin = -43.954410552978516 -ymax = -43.953250885009766 +xmax = 30.05359 +xmin = 30.048397 +ymax = -25.708696 +ymin = -25.713358 [[examples.Segment.sources]] -property = "" dataset = "OpenStreetMap" -record_id = "w53435546@6" -update_time = "2021-05-03T06:37:03Z" +license = "ODbL-1.0" +property = "/routes" +record_id = "r1808544@180" + +[[examples.Segment.sources]] +dataset = "OpenStreetMap" +license = "ODbL-1.0" +property = "/routes" +record_id = "r1808545@177" + +[[examples.Segment.sources]] +dataset = "OpenStreetMap" +license = "ODbL-1.0" +property = "" +record_id = "w338134264@15" +update_time = "2025-04-21T09:53:35Z" [examples.Segment.names] -primary = "Meteorological Lane" +primary = "Vermooten Street" [[examples.Segment.names.rules]] +value = "Vermooten Street" variant = "common" -value = "Meteorological Lane" [[examples.Segment.connectors]] -connector_id = "15b2c131-9137-4add-88c6-2acd3fa61355" at = 0.0 +connector_id = "73a46c48-dc5a-4162-b9c8-1643298784c3" + +[[examples.Segment.connectors]] +at = 0.154695182 +connector_id = "e81188ed-9b2f-48b4-99f2-d894044d88f5" [[examples.Segment.connectors]] -connector_id = "23ae2702-ef77-4d2e-b39d-77360b696d20" -at = 0.523536154 +at = 0.483463065 +connector_id = "11124794-8830-4aff-bd09-f578d3a196b1" + +[[examples.Segment.connectors]] +at = 0.753014135 +connector_id = "cfda5f80-ffe6-4b5c-b219-4f208e0a3832" [[examples.Segment.connectors]] -connector_id = "8e944ce1-4b81-49eb-a823-7d98779c855c" at = 1.0 +connector_id = "a5871213-947e-4342-b486-f560f0ac22f3" + +[[examples.Segment.access_restrictions]] +access_type = "denied" + +[examples.Segment.access_restrictions.when] + +[[examples.Segment.access_restrictions.when.vehicle]] +comparison = "greater_than" +dimension = "height" +unit = "m" +value = 5.2 + +[[examples.Segment.speed_limits]] + +[examples.Segment.speed_limits.max_speed] +unit = "km/h" +value = 60 + +[examples.Segment.speed_limits.when] +heading = "forward" [[examples.Segment.road_surface]] -value = "gravel" +value = "paved" + +[[examples.Segment.routes]] +name = "R33 (northbound)" +network = "za:regional" +ref = "R33" + +[[examples.Segment.routes]] +name = "R33 (southbound)" +network = "za:regional" +ref = "R33" + +# Rail segment: disused railway, Mpulungu, Zambia (2026-02-18.0) +[[examples.Segment]] +class = "unknown" +geometry = "LINESTRING (30.9844394 -12.7185733, 30.9818611 -12.7207838, 30.9815908 -12.7210751)" +id = "2a9415ed-fa07-4734-9d8e-1d8ff69451c2" +subtype = "rail" +theme = "transportation" +type = "segment" +version = 1 + +[examples.Segment.bbox] +xmax = 30.98444 +xmin = 30.98159 +ymax = -12.718572 +ymin = -12.721077 + +[[examples.Segment.sources]] +dataset = "OpenStreetMap" +license = "ODbL-1.0" +property = "" +record_id = "w414442537@2" +update_time = "2026-02-05T14:25:06Z" + +[[examples.Segment.connectors]] +at = 0.0 +connector_id = "2da12352-29c5-479e-932f-68fbe90c8229" + +[[examples.Segment.connectors]] +at = 0.895049489 +connector_id = "feed87bb-7abf-4254-9e14-efd6bdb3e428" + +[[examples.Segment.connectors]] +at = 1.0 +connector_id = "e37ca4ff-ab09-4c84-8d3c-450c703d7308" + +[[examples.Segment.rail_flags]] +values = ["is_disused"] diff --git a/packages/overture-schema-transportation-theme/src/overture/schema/transportation/models.py b/packages/overture-schema-transportation-theme/src/overture/schema/transportation/models.py index 0d2685df9..260fd4574 100644 --- a/packages/overture-schema-transportation-theme/src/overture/schema/transportation/models.py +++ b/packages/overture-schema-transportation-theme/src/overture/schema/transportation/models.py @@ -37,7 +37,7 @@ def _connector_type() -> type[OvertureFeature]: - from .connector import Connector + from .connector import Connector # noqa: PLC0415 return Connector diff --git a/packages/overture-schema-transportation-theme/tests/segment_baseline_schema.json b/packages/overture-schema-transportation-theme/tests/segment_baseline_schema.json index d121e69c4..d4e31ffab 100644 --- a/packages/overture-schema-transportation-theme/tests/segment_baseline_schema.json +++ b/packages/overture-schema-transportation-theme/tests/segment_baseline_schema.json @@ -1883,7 +1883,18 @@ "vehicle": { "description": "A list of one or more vehicle parameters that limit the vehicles the containing AccessRestrictionRule applies to.", "items": { - "anyOf": [ + "description": "Selects vehicles that a scope applies to based on criteria such as height, weight, or axle count.", + "discriminator": { + "mapping": { + "axle_count": "#/$defs/VehicleAxleCountSelector", + "height": "#/$defs/VehicleHeightSelector", + "length": "#/$defs/VehicleLengthSelector", + "weight": "#/$defs/VehicleWeightSelector", + "width": "#/$defs/VehicleWidthSelector" + }, + "propertyName": "dimension" + }, + "oneOf": [ { "$ref": "#/$defs/VehicleAxleCountSelector" }, @@ -1899,8 +1910,7 @@ { "$ref": "#/$defs/VehicleWidthSelector" } - ], - "description": "Selects vehicles that a scope applies to based on criteria such as height, weight, or axle count." + ] }, "minItems": 1, "title": "Vehicle", @@ -2046,7 +2056,18 @@ "vehicle": { "description": "A list of one or more vehicle parameters that limit the vehicles the containing ProhibitedTransitionRule applies to.", "items": { - "anyOf": [ + "description": "Selects vehicles that a scope applies to based on criteria such as height, weight, or axle count.", + "discriminator": { + "mapping": { + "axle_count": "#/$defs/VehicleAxleCountSelector", + "height": "#/$defs/VehicleHeightSelector", + "length": "#/$defs/VehicleLengthSelector", + "weight": "#/$defs/VehicleWeightSelector", + "width": "#/$defs/VehicleWidthSelector" + }, + "propertyName": "dimension" + }, + "oneOf": [ { "$ref": "#/$defs/VehicleAxleCountSelector" }, @@ -2062,8 +2083,7 @@ { "$ref": "#/$defs/VehicleWidthSelector" } - ], - "description": "Selects vehicles that a scope applies to based on criteria such as height, weight, or axle count." + ] }, "minItems": 1, "title": "Vehicle", @@ -2194,7 +2214,18 @@ "vehicle": { "description": "A list of one or more vehicle parameters that limit the vehicles the containing SpeedLimitRule applies to.", "items": { - "anyOf": [ + "description": "Selects vehicles that a scope applies to based on criteria such as height, weight, or axle count.", + "discriminator": { + "mapping": { + "axle_count": "#/$defs/VehicleAxleCountSelector", + "height": "#/$defs/VehicleHeightSelector", + "length": "#/$defs/VehicleLengthSelector", + "weight": "#/$defs/VehicleWeightSelector", + "width": "#/$defs/VehicleWidthSelector" + }, + "propertyName": "dimension" + }, + "oneOf": [ { "$ref": "#/$defs/VehicleAxleCountSelector" }, @@ -2210,8 +2241,7 @@ { "$ref": "#/$defs/VehicleWidthSelector" } - ], - "description": "Selects vehicles that a scope applies to based on criteria such as height, weight, or axle count." + ] }, "minItems": 1, "title": "Vehicle", diff --git a/pyproject.toml b/pyproject.toml index 6046d76a6..ea86feca4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ select = [ "C4", # flake8-comprehensions "UP", # pyupgrade, "ANN", # flake8-annotations + "PLC0415", # import-outside-top-level (prefer top-level imports) ] [tool.ruff.lint.per-file-ignores] @@ -54,6 +55,7 @@ dev = [ "pydocstyle>=6.3.0", "pytest>=9.0.0", "pytest-cov>=7.0.0", + "pytest-testmon>=2.2.0", "ruff>=0.13.0", ] @@ -71,6 +73,7 @@ pythonpath = [ "packages/overture-schema-common/tests", "packages/overture-schema-divisions-theme/tests", "packages/overture-schema-places-theme/tests", + "packages/overture-schema-pyspark/tests", "packages/overture-schema-system/tests", "packages/overture-schema-transportation-theme/tests", "packages/overture-schema/tests", diff --git a/uv.lock b/uv.lock index eae25d7f1..4c896c25c 100644 --- a/uv.lock +++ b/uv.lock @@ -7,6 +7,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] +[options] +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer-span = "P1W" + [manifest] members = [ "overture-schema", @@ -19,6 +23,7 @@ members = [ "overture-schema-common", "overture-schema-divisions-theme", "overture-schema-places-theme", + "overture-schema-pyspark", "overture-schema-system", "overture-schema-transportation-theme", "overture-schema-workspace", @@ -33,175 +38,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] -[[package]] -name = "ast-serialize" -version = "0.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/81/9d/09e27731bd5864a9ce04e3244074e674bb8936bf62b45e0357248717adac/ast_serialize-0.5.0.tar.gz", hash = "sha256:5880091bfe6f4f986f22866375c2e884843e7a0b6343ae41aeea659613d879b6", size = 61157, upload-time = "2026-05-17T17:48:29.429Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/9a/13dde51ba9e15f8b97957ab7cb0120d0e381524d651c6bd630b9c359227f/ast_serialize-0.5.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8f5c14f169eb0972c0c21bada5358b23d6047c76583b005234f865b11f1fa00a", size = 1183520, upload-time = "2026-05-17T17:47:30.831Z" }, - { url = "https://files.pythonhosted.org/packages/37/de/5a7f0a9fe68944f536632a5af84676739c7d2582be42deb082634bf3a754/ast_serialize-0.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7d1a2de9de5be04652f0ed60738356ef94f66db37924a9499fffe98dc491aa0b", size = 1175779, upload-time = "2026-05-17T17:47:32.551Z" }, - { url = "https://files.pythonhosted.org/packages/9c/81/0bb853e76e4f6e9a1855d569003c59e19ffac45f7079d91505d1bb212f92/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be5173fb66f9b49026d9d5a2ff0fc7c7009077107c0eb285b2d60fdf1fe10bd1", size = 1233750, upload-time = "2026-05-17T17:47:34.731Z" }, - { url = "https://files.pythonhosted.org/packages/e5/d3/4cf705beeccc08754d0bbda99aefff26110e209b9a07ac8a6b60eec48531/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f8015cd071ac1339924ee2b8098c93e00e155f30a16f40ec9816fcf84f4753f6", size = 1235942, upload-time = "2026-05-17T17:47:36.287Z" }, - { url = "https://files.pythonhosted.org/packages/26/c8/ee097e437ea27dd2b8b227865c875492b585650a5802a22d82b304c8201b/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5499e8797edff2a9186aa313ed382c6b422e798e9332d9953badcee6e69a88f2", size = 1442517, upload-time = "2026-05-17T17:47:38.17Z" }, - { url = "https://files.pythonhosted.org/packages/ff/bd/68063442838f1ba68ec72b5436430bc75b3bb17a1a3c3063f09b0c05ae2b/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6848f2a093fb5548751a9a09bff8fcd229e2bbeb0e3331f391b6ae6d26cd9903", size = 1254081, upload-time = "2026-05-17T17:47:39.826Z" }, - { url = "https://files.pythonhosted.org/packages/50/e2/1e520793bc6a4e4524a6ab022391e827825eaa0c3811828bfdc6852eca26/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:832d4c998e0b091fd60a6d6bceee535483c4d490de9ba85003af835225719261", size = 1259910, upload-time = "2026-05-17T17:47:41.369Z" }, - { url = "https://files.pythonhosted.org/packages/4e/e1/49b60f467979979cfe6913b43948ff25bca971ad0591d181812f163a988e/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:16db7c62ec0b8efe1d7afd283a388d8f74f2605d56032e5a37747d2de8dba027", size = 1250678, upload-time = "2026-05-17T17:47:43.702Z" }, - { url = "https://files.pythonhosted.org/packages/74/ba/66ab9555de6275677566f6574e5ef6c29cb185ea866f643bc06f8280a8ee/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:baf5eb061eb5bccade4128ad42da33787d72f6013809cd1b590376ece8b3c937", size = 1301603, upload-time = "2026-05-17T17:47:46.256Z" }, - { url = "https://files.pythonhosted.org/packages/66/42/6aca9b9abc710014b2be9059689e5dd1679339e78f567ffb4d255a9e2050/ast_serialize-0.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:104e4a35bd7c124173c41760ef9aaea17ddb3f86c65cb643671d59afbe3ee94c", size = 1410332, upload-time = "2026-05-17T17:47:47.899Z" }, - { url = "https://files.pythonhosted.org/packages/47/68/2f76594432a22581ecf878b5e75a9b8601c24b2241cf0bbeb1e21fcf370c/ast_serialize-0.5.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:36be371028fc1675acb38a331bde160dbab7ff907fdf00b67eb6911aa106951b", size = 1509979, upload-time = "2026-05-17T17:47:50.942Z" }, - { url = "https://files.pythonhosted.org/packages/40/ac/a93c9b58292653f6c595752f677a08e608f903b710594909e9231a389b3b/ast_serialize-0.5.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:061ee58bdb52341c8201a6df41182a977736bae3b7ded87ca7176ca25a8a47ab", size = 1505002, upload-time = "2026-05-17T17:47:54.093Z" }, - { url = "https://files.pythonhosted.org/packages/14/2e/b278f68c497ee2f1d1576cbbef8db5281cd4a5f2db040537592ac9c8862e/ast_serialize-0.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b15219e9cdc9f53f6f4cb51c009203507228226148c05c5e8fe451c28b435eb3", size = 1456231, upload-time = "2026-05-17T17:47:56.311Z" }, - { url = "https://files.pythonhosted.org/packages/0b/43/419be1c566a4c504cd8fd60ce2f84e790f295495c0f327cfaeadf3d51012/ast_serialize-0.5.0-cp314-cp314t-win32.whl", hash = "sha256:842d1c004bb466c7df036f95fabef789570541922b10976b12f5592a69cf0b38", size = 1058668, upload-time = "2026-05-17T17:47:58.305Z" }, - { url = "https://files.pythonhosted.org/packages/03/6f/c9d4d549295ed05111aeb8853232d1afd9d0a179fddb01eeffbb3a4a6842/ast_serialize-0.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b0c06d760909b095cc466356dfccd05a1c7233a6ca191c020dca2c6a6f16c24c", size = 1101075, upload-time = "2026-05-17T17:48:00.35Z" }, - { url = "https://files.pythonhosted.org/packages/d0/8e/d00c5ab30c58222e07d62956fca86c59d91b9ad32997e633c38b526623a3/ast_serialize-0.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:787baedb0262cc49e8ce37cc15c00ae818e46a165a3b36f5e21ed174998104cb", size = 1075347, upload-time = "2026-05-17T17:48:01.753Z" }, - { url = "https://files.pythonhosted.org/packages/e0/9e/dc2530acb3a60dc6e46d65abf27d1d9f86721694757906a148d90a6860de/ast_serialize-0.5.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:0668aa9459cfa8c9c49ddd2163ebcf43088ba045ef7492af6fe22e0098303101", size = 1191380, upload-time = "2026-05-17T17:48:03.738Z" }, - { url = "https://files.pythonhosted.org/packages/26/0a/bd3d18a582f273d6c843d16bb9e22e9e16365ff7991e92f18f798e9f1224/ast_serialize-0.5.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:bf683d6363edf2b39eed6b6d4fe22d34b6203867a67e27134d9e2a2680c4bc4a", size = 1183879, upload-time = "2026-05-17T17:48:05.463Z" }, - { url = "https://files.pythonhosted.org/packages/40/ae/1f919100f8620887af58fcc381c61a1f218cdf89c6e155f87b213e61010a/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc22cf0c9be65e71cf88fda130af60d61eb4a79370ad4cfe7900d48a4aa2211", size = 1244529, upload-time = "2026-05-17T17:48:07.008Z" }, - { url = "https://files.pythonhosted.org/packages/c6/ca/6376559dcce707cdbc1d0d9a13c8d3baaaa501e949ce0ebdc4230cd881aa/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f66173891548c9f2726bf27957b41cabce12fa679dc6da505ddbde4d4b3b31cf", size = 1240560, upload-time = "2026-05-17T17:48:08.46Z" }, - { url = "https://files.pythonhosted.org/packages/35/b2/a620e206b5aeb7efbf2710336df57d457cffbb3991076bbcc1147ef9abd4/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e42d729ef2be96a14efbad355093284739e3670ece3e534f82cc8832790911d9", size = 1451172, upload-time = "2026-05-17T17:48:09.922Z" }, - { url = "https://files.pythonhosted.org/packages/fa/e0/4ad5c04c24a40481b2935ce9a0ccdb6023dc8b667167d06ae530cc3512f2/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b725026bafa801dbd7310eb13a75f0a2e370e7e51b2cb225f9d21fcfadf919ee", size = 1265072, upload-time = "2026-05-17T17:48:11.469Z" }, - { url = "https://files.pythonhosted.org/packages/b2/71/4d1d479aa56d0101c40e17720c3d6ac2af7269ea0487a80b18e7bfd1a5b7/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b54f60c1d78767a53b67eaa663f0dfac3afe606aa07f1301572f588b73d64809", size = 1270488, upload-time = "2026-05-17T17:48:13.575Z" }, - { url = "https://files.pythonhosted.org/packages/6d/4f/0de1bbe06f6edef9fde4ed12ca8e7b3ec7e6e2bd4e672c5af487f7957665/ast_serialize-0.5.0-cp39-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:27d51654fc240a1e87e742d353d98eb45b75f62f129086b3596ab53df2ac2a43", size = 1260702, upload-time = "2026-05-17T17:48:15.141Z" }, - { url = "https://files.pythonhosted.org/packages/75/61/e00872439cfdddcc3c1b6cdaa6e5d904ba8e26a18807c67c4e14409d0ca8/ast_serialize-0.5.0-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c36237c46dd1674542f2109740ea5ea485a169bf1431939ada0434e17934", size = 1311182, upload-time = "2026-05-17T17:48:16.779Z" }, - { url = "https://files.pythonhosted.org/packages/76/8e/699a5b955f7926956c95e9e1d74132acad73c2fe7a426f94da89123c20aa/ast_serialize-0.5.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1943db345233cc7194a470f13afa9c59772c0b123dea0c9414c4d4ca54369759", size = 1421410, upload-time = "2026-05-17T17:48:18.527Z" }, - { url = "https://files.pythonhosted.org/packages/a9/ae/d5b7626874478997adc7a29ab28accf21e596fb590c944290401dfd0b29e/ast_serialize-0.5.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:df1c00022cbbcb064bfaa505aa9c9295362443ce5dacb459d1331d3da353f887", size = 1516587, upload-time = "2026-05-17T17:48:20.133Z" }, - { url = "https://files.pythonhosted.org/packages/0c/ce/b59e02a82d9c4244d64cde502e0b00e83e38816abe19155ceb5437402c7f/ast_serialize-0.5.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:cae65289fc456fde04af979a2be09302ef5d8ab92ef23e596d6746dc267ada27", size = 1515171, upload-time = "2026-05-17T17:48:21.921Z" }, - { url = "https://files.pythonhosted.org/packages/8b/38/d8d90042747d05aa08d4efcf1c99035a5f670a6bf4c214d31644392afbca/ast_serialize-0.5.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:239a4c354e8d676e9d94631d1d4a64edc6b266f86ff3a5a80aedd344f342c01d", size = 1464668, upload-time = "2026-05-17T17:48:23.544Z" }, - { url = "https://files.pythonhosted.org/packages/dd/51/5b840c4df7334104cecffa28f23904fe81ca89ca223d2450e288de39fd3c/ast_serialize-0.5.0-cp39-abi3-win32.whl", hash = "sha256:143a4ef63285a075871908fda3672dc21864b83a8ec3ee12304aa3e4c5387b9a", size = 1068311, upload-time = "2026-05-17T17:48:25.027Z" }, - { url = "https://files.pythonhosted.org/packages/41/11/ca5672c7d491825bc4cd6702dea106a6b60d928707712ec257c7833ae476/ast_serialize-0.5.0-cp39-abi3-win_amd64.whl", hash = "sha256:cf25572c526add400f26a4750dc6ce0c3bb93fc1f75e7ae0cad4ce4f2cd5c590", size = 1108931, upload-time = "2026-05-17T17:48:26.591Z" }, - { url = "https://files.pythonhosted.org/packages/45/19/cc8bd127d28a43da249aa955cfd164cf8fd534e79e42cea96c4854d72fd0/ast_serialize-0.5.0-cp39-abi3-win_arm64.whl", hash = "sha256:92a31c9c20d25a076edaeec76b128a3535d74a24f340b9a8a7e96c9b86dc9642", size = 1081181, upload-time = "2026-05-17T17:48:28.122Z" }, -] - -[[package]] -name = "cachebox" -version = "5.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/36/f6/85f176d2518cf1d1be5f981fc2dadf6b131e33fefd721f36b330e3434d6c/cachebox-5.2.3.tar.gz", hash = "sha256:b1f68246685aa739bbbd2734befb1465363a1e1042407c154feadb065f17a099", size = 63686, upload-time = "2026-04-10T12:21:35.028Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/9e/88193fcb7a2a43fe8ed9d9888374d43fa5c7176aa802651e68b28f1aee4a/cachebox-5.2.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c2c89720547271d36e10cad2c7302bbe11f46eb39eead0a2c321c2d371b8f8b6", size = 374393, upload-time = "2026-04-10T12:20:20.424Z" }, - { url = "https://files.pythonhosted.org/packages/98/8d/e0b13d9bfd43f295cce7824ebaac1970f818a7027c16f290de404934cafe/cachebox-5.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e7f33d24e90dc8aa26762e25898c91a1223b66685420a28a3628fa2e006924f5", size = 356318, upload-time = "2026-04-10T12:20:07.518Z" }, - { url = "https://files.pythonhosted.org/packages/bc/02/8ae1b63dbdebb2ebf600523f48b54e9bfb10db5a28551c3432346f49e1dd/cachebox-5.2.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56cb03ec6289a2ac5daf7422d755683324f02d821bfa796087100df2a7ebd5de", size = 395782, upload-time = "2026-04-10T12:18:50.054Z" }, - { url = "https://files.pythonhosted.org/packages/e4/2f/79a8a0057f354581c25a1a00ddabbd5db4b8631d192670d7a0cc4271dbb7/cachebox-5.2.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a71a71df463ba4c86bc843fa01c3a2a721033adefad888af28c6b65e1915a75c", size = 353194, upload-time = "2026-04-10T12:19:03.083Z" }, - { url = "https://files.pythonhosted.org/packages/3b/57/a1fead35cf481432bd87def0653cd4a069b1ea5847589255795e49ae74b8/cachebox-5.2.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bbe4655371d19fc9f4f5874312bcb6e5b5b6182989979ac33d93c34c8d10c012", size = 371090, upload-time = "2026-04-10T12:19:16.019Z" }, - { url = "https://files.pythonhosted.org/packages/8c/58/53f1fab8bcc3238fd6c533ef3ab146097986a8acb722863c688a2410c1b2/cachebox-5.2.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4974476d1779961df89d6e6f79e6103a1659289d3ee11c92adcb52e236a8aaeb", size = 390902, upload-time = "2026-04-10T12:19:28.258Z" }, - { url = "https://files.pythonhosted.org/packages/11/2f/5abff74666f8388d2c9516c265f99c33484c827f7fcb3cd703c2f3cbb17e/cachebox-5.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad16d733219f4cab3eec6533af30ab7b9c919c6e3e22ad1ef4eb82629a62edef", size = 395855, upload-time = "2026-04-10T12:19:54.207Z" }, - { url = "https://files.pythonhosted.org/packages/dd/11/30b429db12ab5df663aa108bcfac42805f733da65b0bf452f60bfaf4a530/cachebox-5.2.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:12a9e0a93774ca2b3a9fe8a2a0d0812e399fac4af0fce6246a5bca1e7009b8fc", size = 425760, upload-time = "2026-04-10T12:19:41.138Z" }, - { url = "https://files.pythonhosted.org/packages/cd/b4/fdac1bb902b954c03d23eb301d645a328c9664caff5898930fdbd92fde80/cachebox-5.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:be89497a011eb7a638d13cc520244d77579c0f515b95bf759b3de0b90a015203", size = 564988, upload-time = "2026-04-10T12:20:34.673Z" }, - { url = "https://files.pythonhosted.org/packages/4e/63/76cd5405b0339f15bf86593258bf9bc5608f10a5e0fa6f37a282b42a6caa/cachebox-5.2.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:dd01fc0c1934cccb76493eb4b149a9232d299e5e0275f557adf875c3d25cec81", size = 669110, upload-time = "2026-04-10T12:20:49.039Z" }, - { url = "https://files.pythonhosted.org/packages/d9/bc/52d154aa0407bafce94d1d8d3ff27ca5e842f8311be43cfabdefcbb0f6b7/cachebox-5.2.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:a0dfd97b0968f8bd48c33098a03d10f797964559c3a437c84bf97a9973545714", size = 643768, upload-time = "2026-04-10T12:21:04.095Z" }, - { url = "https://files.pythonhosted.org/packages/51/d9/82627eb8cecaf5e7e601bbc65d474a1c3053a2fbc21618ddc6aac19c47dc/cachebox-5.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:223ccf7ac60f595def258e7bc74c0b1d6f43991c9cae6d06749c803d22786d99", size = 610047, upload-time = "2026-04-10T12:21:19.469Z" }, - { url = "https://files.pythonhosted.org/packages/6e/2e/cc5b303746418fde00c93ddbc295733b4e2d131d2e8f5afbc6f45f50454e/cachebox-5.2.3-cp310-cp310-win32.whl", hash = "sha256:745b805fdd99931c3ce1d87d2ee21ca3fb62cba6b4e1f674907af87aad73dce4", size = 275529, upload-time = "2026-04-10T12:21:49.84Z" }, - { url = "https://files.pythonhosted.org/packages/31/72/fb10d6f779d041f701b89f0b7830329f51d1846fbc600869f9f7d635b7b5/cachebox-5.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:a87b19c0a3d8d665a9805b5b4afd64b40082395b70ebe2756131ed1edb0c8f02", size = 287988, upload-time = "2026-04-10T12:21:36.41Z" }, - { url = "https://files.pythonhosted.org/packages/81/88/154179d492f2c000fe6efab3c3ff6b8eb94fbfaa09efe47999bce6b1e29f/cachebox-5.2.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:996f49d04b234082530afcc650bdd00556afbebc19c6c0daaafb85950340cb3c", size = 374245, upload-time = "2026-04-10T12:20:22.042Z" }, - { url = "https://files.pythonhosted.org/packages/7d/9d/3b03f2e063161bcb1a5e0969d521b5c622c2da02252a5c8bd4ef0e4f9914/cachebox-5.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:23a3300ebbb526fa12ce6fa53699002f5fba6da23b4bbbaf8ba8b18a3f03e6b3", size = 356308, upload-time = "2026-04-10T12:20:09.149Z" }, - { url = "https://files.pythonhosted.org/packages/bb/9b/8da38af731e3832e9f987548e4bfb610d7f3054019e12c44a94ba9272b37/cachebox-5.2.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79c63ee1589364caa04c018405e625d2e44e0bf9994f2715b2f322075d8c45b6", size = 395666, upload-time = "2026-04-10T12:18:51.89Z" }, - { url = "https://files.pythonhosted.org/packages/01/dd/1522aa808f94c904c5eb3640991799fed14dd43c1dd99a9f7b71bd95b1e3/cachebox-5.2.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ebd0f8d4ebc3943c1ddcbbdc54f1a8ddf95505c862ed5731319cebd1eb98ae41", size = 353362, upload-time = "2026-04-10T12:19:04.536Z" }, - { url = "https://files.pythonhosted.org/packages/dd/52/95bf883ec9b69a76f3a7d9fb14d015d9a4bdab0143a3eff62ceebc8b1419/cachebox-5.2.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:569966efcc6309aa7d774443e3513cdbb8671efae0158138ba2ebb7d8cc9d8ed", size = 371007, upload-time = "2026-04-10T12:19:17.484Z" }, - { url = "https://files.pythonhosted.org/packages/4b/3d/cc02066d5ccfcb8b35adbaf867977fdb54572cda56ace56da396f0caa3bf/cachebox-5.2.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5774d06f0da37dd566239a4376d6ca8cf983d3e4c3228712ec22b4130f662f21", size = 390670, upload-time = "2026-04-10T12:19:29.685Z" }, - { url = "https://files.pythonhosted.org/packages/b3/50/8e4d59b3e344405d8393d6cc5cc92754d3cc1d81134041ebffd3f5ab73e6/cachebox-5.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae5bf8755bc66bcf42e7ca5c42d703a041a7aaad58f9a0c3be54d5b1cefd2641", size = 395765, upload-time = "2026-04-10T12:19:56.169Z" }, - { url = "https://files.pythonhosted.org/packages/e5/d4/d731cff1c4cec22404bd3ddda05b233c5efaa5f13d7abf4e2728905b7cdd/cachebox-5.2.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:63f061cc6a5ca70bbce2e6be0588fe2fee00a93a1b0581b1086d54b10288cdb6", size = 425707, upload-time = "2026-04-10T12:19:42.714Z" }, - { url = "https://files.pythonhosted.org/packages/36/01/3ec8aadceb0dcc66dbd0b9b32966cf7b6928ed84471424c24d21b0af62d0/cachebox-5.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:577c781f18b559f4dc9eea176c6aed008843ef4b8e045cf61bb519e09dccc9ef", size = 564759, upload-time = "2026-04-10T12:20:36.268Z" }, - { url = "https://files.pythonhosted.org/packages/db/23/31cbc8623ecc2e25900f7e8f20f11bfb84786989a59a8046e70b27cbea6d/cachebox-5.2.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:7f691e25572a3ddbb018e19d796f774713bd6b0f7ce9be2e71f6e18572de264a", size = 669309, upload-time = "2026-04-10T12:20:51.117Z" }, - { url = "https://files.pythonhosted.org/packages/34/29/5a9e92bdc7b32dc865e73dd776638244f900136daee5bb0591a67e1530fa/cachebox-5.2.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:33368adf86669c29b936fbae5d6219cf90aacd4b1db71dae2e23d584a8219cd6", size = 643705, upload-time = "2026-04-10T12:21:05.882Z" }, - { url = "https://files.pythonhosted.org/packages/04/90/5273a412855fdc11f674e4749aee6d5ec0a91f5c1a9f6e922f7fa0cb7a83/cachebox-5.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38ce67b7b45713e49459a09411d07f82de04022c04aecde6202cd32f934c2b1f", size = 609751, upload-time = "2026-04-10T12:21:21.331Z" }, - { url = "https://files.pythonhosted.org/packages/a1/a4/0fadb5e6a00f373cc3fe56b4415cdea2fc0147f6ec475611762d16eb4b05/cachebox-5.2.3-cp311-cp311-win32.whl", hash = "sha256:a7cd2c81347063ab6c512d0f569aeb5f75fc2dfe686c8486258ffd08052324f4", size = 275485, upload-time = "2026-04-10T12:21:51.563Z" }, - { url = "https://files.pythonhosted.org/packages/03/83/67c1bf83f815294d2c3acd7631f25b5cbe6067e1d56495f76829dd60057b/cachebox-5.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:7e45798d6b969794840bb302857946d710ecb32af78dfcb3ab40f4e68ee7fdaf", size = 288024, upload-time = "2026-04-10T12:21:37.999Z" }, - { url = "https://files.pythonhosted.org/packages/e4/e7/6fa6abfc9c4c07b88f09a88466fa93c7081fd679d8e06f8f558bb4ac845c/cachebox-5.2.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:09c0340e9daa7b4530801e5a570cb0c1a1ad941a85d245d360020d3986d0e787", size = 377791, upload-time = "2026-04-10T12:20:23.87Z" }, - { url = "https://files.pythonhosted.org/packages/3a/79/89e4423352d0ca33bbf80fc1b4b665e654a93de8b16cf41e96fcac81801a/cachebox-5.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3162758792626685ec34950eedd565d015b115d0ff0d751d2716031fc32d51b", size = 359562, upload-time = "2026-04-10T12:20:10.626Z" }, - { url = "https://files.pythonhosted.org/packages/d2/ab/e533c2751e6a3411ebe369277aaed03199b9e4586a48f0a3712a1f4b418b/cachebox-5.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a189a780c3ccd7b9d157074ba6bf3e191e522b39abbdb590075111851f02d50d", size = 397910, upload-time = "2026-04-10T12:18:53.336Z" }, - { url = "https://files.pythonhosted.org/packages/7a/0d/b8492d6ca53278499a37c9f9d51afd4ad77bfbe813d6281944d45b97a1e7/cachebox-5.2.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:410b67baa99d433644199b11289627f7ebba4ee5786f95ca9858f238afcee157", size = 353699, upload-time = "2026-04-10T12:19:06.248Z" }, - { url = "https://files.pythonhosted.org/packages/78/d4/fd20b3a5362651303fa12d3ee62f56af2bd396e4a7303d7014a1a1e5b392/cachebox-5.2.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f81474dc19d3865fa5e57263f834bc6bbc00e471a594fb9d934ed552732c02fd", size = 372510, upload-time = "2026-04-10T12:19:18.997Z" }, - { url = "https://files.pythonhosted.org/packages/71/94/3ec55c946d300cc4eaed3a0f79740051ac6e11ef4032421332c6ca15f5d5/cachebox-5.2.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:85ccd827193b3e3e887a88a16b88ef7ed174e7e65be515b5253322aa75e665c3", size = 392802, upload-time = "2026-04-10T12:19:31.196Z" }, - { url = "https://files.pythonhosted.org/packages/01/b1/1a3c4e436ad8a4c4ba3e70f4c62e1f927cbbb3c943a9bba5813b8b815bde/cachebox-5.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a1e7d3cb8a5e7e68996a8619e3ef8771a124d14568c251f9e586eba88d759c1", size = 398223, upload-time = "2026-04-10T12:19:57.583Z" }, - { url = "https://files.pythonhosted.org/packages/0a/ea/d36ad3976c4396b350b96a1582411b7a00e56c144eec0bb5ba5f36ce7d86/cachebox-5.2.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:adcedfcfcb933b21e7fdcfe560c79887bc8287abceab0586aa3730417dd0277d", size = 427696, upload-time = "2026-04-10T12:19:44.361Z" }, - { url = "https://files.pythonhosted.org/packages/a8/36/71845b5c7a9ffbd85e6fdb470c11a174f499bd5238fa37b1214157c2454d/cachebox-5.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c7f0c72c51a3a9e7049ea6ff2a43cd3877ab7fee966eb65771a59621563b75e3", size = 567854, upload-time = "2026-04-10T12:20:38.357Z" }, - { url = "https://files.pythonhosted.org/packages/e8/a2/baf0e5a8392e64e352b137ccd7356b3d98068c842fd19f510a7790c05d34/cachebox-5.2.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c48c10e498d573511aafbd545570e7f43b40a7428dc282183bf5adc334d9e1a8", size = 670306, upload-time = "2026-04-10T12:20:52.903Z" }, - { url = "https://files.pythonhosted.org/packages/a5/22/cd4e4c1d624b8ef9fb4b8bebf0bf5d2d74a399cf1ac46b667bb79d15359a/cachebox-5.2.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2f1e086ab5ffd082a68bb63699d517655a59b06414927bfc84e01df91b81e34d", size = 645943, upload-time = "2026-04-10T12:21:08.238Z" }, - { url = "https://files.pythonhosted.org/packages/0a/d6/55859981f5ec6a9e412baaa4db6aa5973a00008750b3f054cdefcb6491fc/cachebox-5.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:649d18399f13735bb82daa33800196f815529c49e967767c40ca221723e68afa", size = 612309, upload-time = "2026-04-10T12:21:23.404Z" }, - { url = "https://files.pythonhosted.org/packages/d7/1e/313f650467ac85824c4199188f8f1ee3386cd12eb665dbf7c88d372e4956/cachebox-5.2.3-cp312-cp312-win32.whl", hash = "sha256:0a17aeb4e5b1c6ef1c3db8fc5186f9986e215ba5ea5a5d08baa45bcf55f261b2", size = 279789, upload-time = "2026-04-10T12:21:53.215Z" }, - { url = "https://files.pythonhosted.org/packages/c5/50/3b334f887accfa811cf5c7533b8ce22c523eb009363a86401198899dadd2/cachebox-5.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:cfd69114141ab362acaa2099e425a1b965cf7b021a539a4e953143d593930b74", size = 290917, upload-time = "2026-04-10T12:21:39.696Z" }, - { url = "https://files.pythonhosted.org/packages/31/3b/16d5c295f6ec2913ef595b39986dc7b7cc179fdd2e73f5ebd1814c38fd51/cachebox-5.2.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9527c5c70f8735f2d696331d8bcf77254f03b4dc8542046807823bd36ed4e8ba", size = 377408, upload-time = "2026-04-10T12:20:25.444Z" }, - { url = "https://files.pythonhosted.org/packages/cd/87/45f834154f79721e5b64a80ffab4f9710834c4f9c01fa977f94a9116c32a/cachebox-5.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40ac878af00d5969862c1f6bc076de1e34ca248662fce6aecca1761f52e33e32", size = 359274, upload-time = "2026-04-10T12:20:12.127Z" }, - { url = "https://files.pythonhosted.org/packages/46/17/794e5f93e0a172aa14ecd692f6d89bdf094f71eb35fa923d0a0af25cef1c/cachebox-5.2.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5ff26bfd8f7e95b3becf6d5f65c25edaca50fa68078868648b70d79bcccc260", size = 397520, upload-time = "2026-04-10T12:18:54.807Z" }, - { url = "https://files.pythonhosted.org/packages/23/19/9470b1a96de6e480192b1a92b2fafa72aa052efc2509a5418a5652205b33/cachebox-5.2.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:82e7002dd343afeeba2fcf0e483131b342a27ec3bc34b2214dc617691bda40d6", size = 353183, upload-time = "2026-04-10T12:19:07.797Z" }, - { url = "https://files.pythonhosted.org/packages/6c/2b/72813f80397ed4640e337cbd1a14ab7eaafe33e479291d3623b6a6a55fec/cachebox-5.2.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ccbdc54a6c4b5758408c1083bdfa217bd382894a8331c7d0a54b84ba0cf51e5b", size = 372239, upload-time = "2026-04-10T12:19:20.44Z" }, - { url = "https://files.pythonhosted.org/packages/05/17/47dc9687288fa55486573627089ecd9aae124de5924a4bce008af96d80b6/cachebox-5.2.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df5135a168f143d186b1cc3be0ca16b66446897ab5cedc03bd80bcc926fcd403", size = 392568, upload-time = "2026-04-10T12:19:32.73Z" }, - { url = "https://files.pythonhosted.org/packages/13/95/450765b971a3bed9d7cf003c3833c1976482eb83b0241b6dbb840a25b43b/cachebox-5.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10bedf96db8f9766cc956f9adcc623e604264e5d6fa2e255432f8c2ed7519143", size = 397920, upload-time = "2026-04-10T12:19:59.314Z" }, - { url = "https://files.pythonhosted.org/packages/5f/3e/dd8f4c1f92e58d479913ce9cbaa3227c911128e6046c82f4fd44309f685a/cachebox-5.2.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f22732d0d69bb84ad2dca7480bffdfd0430c647152d488936e152ecbbfee52fb", size = 427332, upload-time = "2026-04-10T12:19:45.888Z" }, - { url = "https://files.pythonhosted.org/packages/7e/20/80d8c26ce63e78da3874a5bb07a3a78de53a2b0356ba80583a4927f0a074/cachebox-5.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:26ae0b68979204d360327f4c0725cfdc95cfc34ab73ab1a8f528e3bd2f6d023c", size = 567494, upload-time = "2026-04-10T12:20:40.373Z" }, - { url = "https://files.pythonhosted.org/packages/10/35/7249885dfed3602b3b48c1e67781197dcdc536c50f72caeabe3944348af8/cachebox-5.2.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:f3d628b816e28a6e7661d460e02dd5b421247cc2cd275814f80ea79621245fc4", size = 669968, upload-time = "2026-04-10T12:20:55.155Z" }, - { url = "https://files.pythonhosted.org/packages/2d/8a/e5b58f0bbd6fef74da5d8e5ab49e67898ce7e6df28c16280a0f2b78461f7/cachebox-5.2.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:64057caa6b741320655cd3c5997fe642dae5dbff571eb530e6f53e58272bb43b", size = 645547, upload-time = "2026-04-10T12:21:09.948Z" }, - { url = "https://files.pythonhosted.org/packages/d8/25/51783a4c6f25ca87ef1b4b762ff0364bd98053a02d597b30d26ff4cf13c5/cachebox-5.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa325306084aa2dc0b21e07723d7700f4d43dece3732c7fdaf7a269dc5e35aa7", size = 611844, upload-time = "2026-04-10T12:21:25.286Z" }, - { url = "https://files.pythonhosted.org/packages/c5/c5/b26c4b046e296d0e249448fe297626b3caca2e851837712f03c358662cb7/cachebox-5.2.3-cp313-cp313-win32.whl", hash = "sha256:55003089d21c2f5515089c307be063b45558e884a4a1cc9593944374c89975c4", size = 279421, upload-time = "2026-04-10T12:21:54.921Z" }, - { url = "https://files.pythonhosted.org/packages/e0/7f/a49420670393bfea618de7a893d45cae9294cf3293d7b158e7af20e8f39e/cachebox-5.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:dcc5edb6ecf2b516e90b773d232360c5e4ed8fdcda038b19441da2ed9cf208ab", size = 290702, upload-time = "2026-04-10T12:21:41.458Z" }, - { url = "https://files.pythonhosted.org/packages/c9/0b/bf83bda13ef6fc490d208a1d4dd712034624526a88f61713cca0edc9884f/cachebox-5.2.3-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:a4b7559fa4994c4032dd07466c2041d57e055feb814762e1f73f4e8beef188d0", size = 371704, upload-time = "2026-04-10T12:20:27.253Z" }, - { url = "https://files.pythonhosted.org/packages/8e/ea/aa5162273238e84f9e41b33600c69299572dc1c8f0f768d07660b71be07d/cachebox-5.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f57afada3d9327adf87f3b5cf0094348c6fd49354ab2e9bd20b044648eb094ae", size = 353385, upload-time = "2026-04-10T12:20:13.668Z" }, - { url = "https://files.pythonhosted.org/packages/47/96/3ca013e2e48df5c1d7855669b208f4bf8014ccb842ccf7a3a0eaac07bee0/cachebox-5.2.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8342ff350ce86f062492752d612e9f056ac5dc56375713d75c3bf6e83b4d18db", size = 392181, upload-time = "2026-04-10T12:18:56.385Z" }, - { url = "https://files.pythonhosted.org/packages/63/ca/1bacb4efa0b0ce8065d1fb7c8dc7c382ec4e1cc3f007eb08417732be2725/cachebox-5.2.3-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:405f9cc8492fc9d953b5a6b9e2b661e99583755c6639ab8d09a287fdf336503c", size = 349494, upload-time = "2026-04-10T12:19:09.505Z" }, - { url = "https://files.pythonhosted.org/packages/d7/2e/75db4bda3768658f5baa5a54f6a4f643bc2de1a16788e40581a080e803c7/cachebox-5.2.3-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94aae393ec1d9b26565d346445bb6afa3963d2a0d3eb5e4188d0e510fab871a0", size = 369216, upload-time = "2026-04-10T12:19:22.224Z" }, - { url = "https://files.pythonhosted.org/packages/f5/82/e1f833be0d57e29a8c5eb0a0275cd34b962f3c7f5b9e0517ec4bf75e7cc3/cachebox-5.2.3-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8b0b575066fc09f6fae0d4bd30d6ff56584a6870cbe7d202916c5e0d725cfd4", size = 385922, upload-time = "2026-04-10T12:19:34.198Z" }, - { url = "https://files.pythonhosted.org/packages/53/d6/615a3c16c1d63839f2c67644eb414c4dc9769ab2e169d935110fd8e268d5/cachebox-5.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41e99c1240106d39b63ce7868a6cd8c9da9243fef08848b85d428164e0769fd2", size = 393276, upload-time = "2026-04-10T12:20:00.925Z" }, - { url = "https://files.pythonhosted.org/packages/2f/a6/7844c9c84b170dae1005b22da174639968e64c8055d66a209a1598663771/cachebox-5.2.3-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:432ca62b99f7eafc21af669d76c88c1b7377db179b89fb6fca3ea93b8f9fff19", size = 421355, upload-time = "2026-04-10T12:19:47.691Z" }, - { url = "https://files.pythonhosted.org/packages/c9/0f/43f62355846cae3dc41cb4daccac0a4bb2b7b8b3c7d77d1b6a220bae6d54/cachebox-5.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e51d9c59006b53447f806145406eb37a7fc3c25553d4fd24c3887f3b268d214e", size = 561656, upload-time = "2026-04-10T12:20:42.161Z" }, - { url = "https://files.pythonhosted.org/packages/9b/fc/a453813c6d000d69a41a06c6a3143a6c4d0d0e41f23c155db2f82ea0edfa/cachebox-5.2.3-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:5e48a405f699fb001b8af120a6e0b4a981277f84eb5dd66a1faa21e4b6fe9485", size = 665791, upload-time = "2026-04-10T12:20:56.842Z" }, - { url = "https://files.pythonhosted.org/packages/aa/a3/f6a9e75f1e602b67b6d67088a9a766adfc4e0a740a9c4b68e4e6207c1006/cachebox-5.2.3-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8cbfc007ea78af61d75d7d26e5854df53dc5da6877d074afd4b4696c074f4ee7", size = 640975, upload-time = "2026-04-10T12:21:11.641Z" }, - { url = "https://files.pythonhosted.org/packages/a3/15/4ac98277f7fd9d855c8ed337e8e2a3386d17997cce2dd3eadb23dedc08e3/cachebox-5.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6a94d0da8133b3a0707ae11c9ea321f8fc37e3b5a14517019a05d632218b0f56", size = 607242, upload-time = "2026-04-10T12:21:27.27Z" }, - { url = "https://files.pythonhosted.org/packages/9c/0b/ce61907a803f75854e0cc91b84c16e14dce0e4e939efbda26293eb4c8784/cachebox-5.2.3-cp313-cp313t-win32.whl", hash = "sha256:5fee33549877c03c2494ec5359a57a7667f872fe8e296a7f39d3dfe08dd3914c", size = 271619, upload-time = "2026-04-10T12:21:56.768Z" }, - { url = "https://files.pythonhosted.org/packages/b0/06/fece190ad5173d06b2779494aaad5528907f2e55c809618e5b67c2e3dbb5/cachebox-5.2.3-cp313-cp313t-win_amd64.whl", hash = "sha256:67548a05cd41fcc4f7af80a2f97f742fef3d436537ac2e1a1dce0fcba5d41190", size = 283133, upload-time = "2026-04-10T12:21:43.037Z" }, - { url = "https://files.pythonhosted.org/packages/b8/8b/72c0e80aad08e09867ce14a621bce689a733552f20cdf2ef96d4b052da10/cachebox-5.2.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:37fa0891f0defee053c09f5f43f802f731e36e6e6ca055d7d174af07f77232ca", size = 380523, upload-time = "2026-04-10T12:20:29.345Z" }, - { url = "https://files.pythonhosted.org/packages/fc/62/33aaade81b181d5191cc39c867c297aa7c65f3191aa9749bf99b77496b88/cachebox-5.2.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dc6315902f2ef4afbf10bc8e08c54ff34de5ce124546b8e0016c9b0d327be21e", size = 362424, upload-time = "2026-04-10T12:20:15.215Z" }, - { url = "https://files.pythonhosted.org/packages/9e/0b/3eedaf9ea4b41c931f4340bfa42056efe2bb5fe3a79649d6c8a1dce585a5/cachebox-5.2.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7df1735ca778480d51b8232fed397ffe3935158f20d34fb1c5ed171b53d5a6e2", size = 399572, upload-time = "2026-04-10T12:18:58.331Z" }, - { url = "https://files.pythonhosted.org/packages/be/69/c79b8a6a5b889ac4a60800bacea3553cb3b86f6fd13b2262bade1cb962c6/cachebox-5.2.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e22451cde8f884051e941b21870e4fc91fcf58d0d8c285bb8964107e1f02445c", size = 353803, upload-time = "2026-04-10T12:19:11.21Z" }, - { url = "https://files.pythonhosted.org/packages/d4/c3/bc7838de51039f8c50506d8dc82f22ff9a652794339a223b12af595e1d2f/cachebox-5.2.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dcbccf3015d9a42bcf41260fa5cc048a5bdb75aa10997d514d6c976117f30ee2", size = 374474, upload-time = "2026-04-10T12:19:23.658Z" }, - { url = "https://files.pythonhosted.org/packages/65/61/e5231ad2ae952ca482f9b9df55df4b96add1a80de28de537c5f574605987/cachebox-5.2.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:311eae5079e256cbbfafdc3dcff1714b6598a767f9c1ef8c3709e74ea0cc12b0", size = 393045, upload-time = "2026-04-10T12:19:35.651Z" }, - { url = "https://files.pythonhosted.org/packages/78/c4/c9b3fa764ac5420a9e079ad53fa8840d4a26b74c4ccda56acbef49cf76ff/cachebox-5.2.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f4d2a80a5cd3380739c67f7d89e596634f5897b8d5a4a3dc1598312cb077535", size = 398700, upload-time = "2026-04-10T12:20:02.513Z" }, - { url = "https://files.pythonhosted.org/packages/9b/3e/c4e3acd4cb04e01c5fb7cc7a4de16059b9594d90672fff85af8670275267/cachebox-5.2.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3977515b727a5203f494c44c4566fb936c4b940351c01d3d8e7b5d104dff4f53", size = 426725, upload-time = "2026-04-10T12:19:49.385Z" }, - { url = "https://files.pythonhosted.org/packages/25/5d/610b79479719951581109d985244d34c97f86a308c3d7c83443e2b1dac46/cachebox-5.2.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c5be17dd5c4fabcfecd5bcf6d54f9c6fb719daed3ef01ac1c03a14af0e2b26c1", size = 570042, upload-time = "2026-04-10T12:20:43.793Z" }, - { url = "https://files.pythonhosted.org/packages/8c/63/cad8a05db4d0c0f5ba6bccb32e57d15c472276de9476f56004445b40711f/cachebox-5.2.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:6d37334fc218fdaee31db8a4f938938716e7c3b1b4059e25de27c8447fc95fde", size = 670974, upload-time = "2026-04-10T12:20:58.528Z" }, - { url = "https://files.pythonhosted.org/packages/54/d1/9cff7c2b9048d1c38b7ad8199ce856596d09720b3bea74043f3bad71970b/cachebox-5.2.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:1e5f1b7e23411b748d919348c3b65db1f9f8927ab8f6f3acae19bd617543df2d", size = 646213, upload-time = "2026-04-10T12:21:13.619Z" }, - { url = "https://files.pythonhosted.org/packages/27/ae/2e1ad162ec13903e84469c8a753baf385f1bc324279d6c7cb6365e7099df/cachebox-5.2.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e7b06a75a898b31fd73c4d8bf727a9b9f8b5b7738cccd0ab5e6fd2a9cf659d3c", size = 612787, upload-time = "2026-04-10T12:21:29.271Z" }, - { url = "https://files.pythonhosted.org/packages/c8/8a/07b5ffd841e1ff534bb6e8721c39fdfe0d7cdaac1398e1783b2a0c37bd22/cachebox-5.2.3-cp314-cp314-win32.whl", hash = "sha256:3b798052719f09a2ce7bf9fa9452dc0a7d4dc53b50a2d3aba6ce6ebc12d39df7", size = 278559, upload-time = "2026-04-10T12:21:58.482Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f8/b88a82ce9ec7a2fa0f09ed1cdd031692c8664c41f9ab71831e177c7ce2df/cachebox-5.2.3-cp314-cp314-win_amd64.whl", hash = "sha256:4afc8b8575e3228a42ad8d819de5fbbecc6bd0b521295966b00244be37ae3b9b", size = 291928, upload-time = "2026-04-10T12:21:44.621Z" }, - { url = "https://files.pythonhosted.org/packages/4a/01/8c79c07c8c6517fb2fe7d479dd87044e38aac5b9af0245b33fcd695eae37/cachebox-5.2.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:0e8a34b82be30d3d9fb7dfaf9a86ec2b3ab9bc264715909ef27fc3d3587324d2", size = 374325, upload-time = "2026-04-10T12:20:30.923Z" }, - { url = "https://files.pythonhosted.org/packages/7f/51/0fc26b923e80ab857ac99d5f7f3784dc941e7b4de361c204835233176ddf/cachebox-5.2.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4d4e336aebf866463878ccd28a4d0ef4003ea216708cf4a02a7f198481b3af81", size = 355444, upload-time = "2026-04-10T12:20:16.879Z" }, - { url = "https://files.pythonhosted.org/packages/c1/6d/a6b399221f8dc4b3e01b37d3240ef5b8a7eb78cd9bfbb99b0e655dd01649/cachebox-5.2.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b102fcdd97b0602bf5d6ba1a571bba3e3d6fa912b89fd768b0da5427408eab8", size = 393978, upload-time = "2026-04-10T12:18:59.753Z" }, - { url = "https://files.pythonhosted.org/packages/bd/f1/4c8f998c117c1941a82bd824d6687280c50167f21fea6392e41531d641e2/cachebox-5.2.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:245a79fb2c5d3bff252f4263f76210ef3ad7c2ff9b0234859b26974830a80491", size = 349298, upload-time = "2026-04-10T12:19:12.843Z" }, - { url = "https://files.pythonhosted.org/packages/d1/dd/683bc5a32a0da660d02fa248b880b71a2b834e9b54b8d272b5801282f402/cachebox-5.2.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd0e8dbd8fd4cf664c645c08f9e10508e133353756705c4a738e90a5406224b5", size = 370619, upload-time = "2026-04-10T12:19:25.298Z" }, - { url = "https://files.pythonhosted.org/packages/81/49/d6c47c78a7769b355076c5b635c2b538c8b88e8ceeb408e104d0f269b515/cachebox-5.2.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdb74294bdc33e39e26606919a9b2229038d5fac0edb80c9056683c08584d4a9", size = 385988, upload-time = "2026-04-10T12:19:37.638Z" }, - { url = "https://files.pythonhosted.org/packages/70/e2/b669555ada7fa1392e4cdb8a19f3367db5c6abef0fde8ab034a9747760df/cachebox-5.2.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bba3e9a7f52fa196b434522f39675f3b32a076976ef2373ded6f1065e99f4d20", size = 394090, upload-time = "2026-04-10T12:20:03.978Z" }, - { url = "https://files.pythonhosted.org/packages/8f/01/42916249e53fe4fcbdf0419fb55dbc09b9f377475376e1d7f4ae9c9bd6cd/cachebox-5.2.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:abb21f0f937fb66528f1b9f1a04874d6aa503e78bbb26f4cf33bf67faddbdd68", size = 421632, upload-time = "2026-04-10T12:19:51.048Z" }, - { url = "https://files.pythonhosted.org/packages/a1/54/34eebe18c6ed8ba27b1331b5e3d08bd8bb62f03ba81fbf47a2db0fa646f7/cachebox-5.2.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:dab6fd3189b0c746fb03e1915fd947aaca9112cedf26ef3a0c39383acf87d2e5", size = 563871, upload-time = "2026-04-10T12:20:45.417Z" }, - { url = "https://files.pythonhosted.org/packages/ba/b2/f92da0d54e4f18609588709090de8c81dd7c8b20ed6ac30f9b91bedbedf5/cachebox-5.2.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4e7d2935b9df11d3717f99c7237b6780f1f8c70e6a99b69b8430d89929ec825", size = 665677, upload-time = "2026-04-10T12:21:00.512Z" }, - { url = "https://files.pythonhosted.org/packages/43/9d/bf2d3dc949afe4d21fc7eb15b7524255e834b9252df6bba111e6686d1c6f/cachebox-5.2.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:611aa260fe1b2506330ff72f415e2cb4053c9c4e3776ac68fe2eedee0e1b91b1", size = 642067, upload-time = "2026-04-10T12:21:15.727Z" }, - { url = "https://files.pythonhosted.org/packages/6e/4f/a789eda189550d239fbaf165b9810f148e733e97a2a4eda7c4192295c7f8/cachebox-5.2.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a424ffb8514a9cb49bacff7995b7c767625cb2239692bd6524245e8579e375cc", size = 608048, upload-time = "2026-04-10T12:21:31.156Z" }, - { url = "https://files.pythonhosted.org/packages/41/c3/590e161c04ffbd36e33933e6dcca5ffa40b5548e3121a21d77aad42af138/cachebox-5.2.3-cp314-cp314t-win32.whl", hash = "sha256:83988dd8e9075ee837e8407e26db49a9944ae74924d5db57b477444d7d98622c", size = 271694, upload-time = "2026-04-10T12:22:00.589Z" }, - { url = "https://files.pythonhosted.org/packages/66/f4/f60b8506df467261178afe918801df37c02c46ec2b8ce019760a14e2abe7/cachebox-5.2.3-cp314-cp314t-win_amd64.whl", hash = "sha256:dbda6390fa5070a19157ae35ab8066d3fe468634e0e9e21452c68ce7999c7d0c", size = 284212, upload-time = "2026-04-10T12:21:46.241Z" }, - { url = "https://files.pythonhosted.org/packages/ce/7b/5eead1ca0d437b1993a742c6571079ae58ae4db50d94d42e87b514aed6c3/cachebox-5.2.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c798cddfb780156db09d3d96ed5da4c2d5fc01dad4bc7b54db5b20c34f221926", size = 376199, upload-time = "2026-04-10T12:20:32.674Z" }, - { url = "https://files.pythonhosted.org/packages/77/e3/5e45042f9b552a5087cafc2e0fed834e632531fca17818201d72e78593ce/cachebox-5.2.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:c8f3de4afeb3fd721620be3d02f2338bcbc3fdbd464ca14e1c474088c9669db0", size = 357109, upload-time = "2026-04-10T12:20:18.554Z" }, - { url = "https://files.pythonhosted.org/packages/d4/51/3c4743b718b42e4b80166fa61f8722b603eba7bf206768a7892c4699dce7/cachebox-5.2.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b39022c258872185327acffa9ad42d6bdf42f37d006d35c825a684eb5fa98d40", size = 396433, upload-time = "2026-04-10T12:19:01.463Z" }, - { url = "https://files.pythonhosted.org/packages/f8/9b/678da91187bdb2836db2b8da62519da75359b46bc28697799a7caa314519/cachebox-5.2.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5a0599fb85dcb6df9a86502435643fe90c793bbcd50b5d85217c70f2bc2e38fc", size = 354287, upload-time = "2026-04-10T12:19:14.55Z" }, - { url = "https://files.pythonhosted.org/packages/df/06/769446da6c9f2855499aaa19e2d7260aa47934bc2e15a931e5b737f8685a/cachebox-5.2.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3cdbe8f1b7716a44dc82ef3a6830a612260c7379478cfa80804632e2e6252b8e", size = 372507, upload-time = "2026-04-10T12:19:26.763Z" }, - { url = "https://files.pythonhosted.org/packages/79/cf/86c60994a7be734abef0395e440dc11714f84ffcd369cbcd8e61c3d58126/cachebox-5.2.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:783d1b9a0b3c77c43e7ae331b9d6561ad75827e16b2484e2a6cc289ec4d392ee", size = 390831, upload-time = "2026-04-10T12:19:39.591Z" }, - { url = "https://files.pythonhosted.org/packages/9d/db/acfb55f8d5ee4ea1c5f2d32ede25d4d04e944ba09d2832c27c085022490d/cachebox-5.2.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c6476a2a842906fee782d92f8fbcb03ecfd22eecc39adb7fb5b047d7e1cf020", size = 396277, upload-time = "2026-04-10T12:20:05.735Z" }, - { url = "https://files.pythonhosted.org/packages/dc/4f/35e27e85a48e15671c5863addcabde910eb311800a621c3e47c04bd36d17/cachebox-5.2.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:184bbcfa1370415b6d1f09e4fb74ab697dac8df09f522aa217a2fac65f973744", size = 426980, upload-time = "2026-04-10T12:19:52.622Z" }, - { url = "https://files.pythonhosted.org/packages/09/4b/50f2cadf20c02db9e449f2e9fee95f3eb5768ab1804dd0a5eba6c98119ad/cachebox-5.2.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:f89df36b46f8f5e11c0c49701ec3cebddf51191f96afb7bb75c394faf3c1cbc8", size = 565539, upload-time = "2026-04-10T12:20:47.051Z" }, - { url = "https://files.pythonhosted.org/packages/43/53/b8e948cadb48b8bcf1d13c2aa4a788ff0e95b50ddb808c18e998499b4680/cachebox-5.2.3-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:fb0bdcd9e28686e3b91d5210c843542858f0f10de151181aee27a7978fe4992e", size = 670870, upload-time = "2026-04-10T12:21:02.141Z" }, - { url = "https://files.pythonhosted.org/packages/29/7b/d68ca3f59a9d6963c2f6b19bc4b1926a37db2e4a4f6c9891d12788e49ce2/cachebox-5.2.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:5196f0d2c2f99c92ddf0d2c37803ff90509d14a5df211b7754feb8b61ffd8740", size = 644542, upload-time = "2026-04-10T12:21:17.541Z" }, - { url = "https://files.pythonhosted.org/packages/f8/c8/44ae6d5dff09f044d61a92591e6a8db17f3b2ee51a54d375cce90271527b/cachebox-5.2.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:73671850d8c3634ab217398c83715d3feb52589ec97bd8e2f4d22e472741ea48", size = 610235, upload-time = "2026-04-10T12:21:32.93Z" }, - { url = "https://files.pythonhosted.org/packages/9a/1b/31cf2449da9a296f6c6c0002c7ae91a25c3a4bfef071763bbeb85300b402/cachebox-5.2.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:70c718f6bb77e6ba142b9a055b81ce85412a0c0e5e82a154489b45e6f91d09ec", size = 287614, upload-time = "2026-04-10T12:21:47.909Z" }, -] - [[package]] name = "click" -version = "8.4.1" +version = "8.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, + { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" }, ] [[package]] @@ -215,115 +61,115 @@ wheels = [ [[package]] name = "coverage" -version = "7.14.1" +version = "7.13.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/fd/0ab2772530e946e1be1abd0bc09e647ec9b02e88f0867857601fefca8953/coverage-7.14.1.tar.gz", hash = "sha256:30c08f7d90415aa98b3c990385dea2939b0da55f38515e5b369b83655f8523be", size = 920132, upload-time = "2026-05-26T20:41:36.783Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", size = 915967, upload-time = "2026-03-17T10:33:18.341Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/92/69/0d2ef01ff4b8fcecd4cba920d11e92fa4f96ae412441d3b56a90a258e69b/coverage-7.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3e3680291c4a1d0dadfa84a2c459576a4af5133abb617905714339a0c73138cf", size = 219722, upload-time = "2026-05-26T20:38:14.002Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ae/9afdeaa31b9d9ce98124b6abf8bb49119bf71aecae04f8567c189d91299f/coverage-7.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a5274669f37f2343635a347b91a60777621341ab3378e9c6ac9335eee704bddf", size = 220240, upload-time = "2026-05-26T20:38:17.424Z" }, - { url = "https://files.pythonhosted.org/packages/51/69/c998589871df7ea7dba865cc5ee32b5a3e1d47ba6c68ef91104c7c46fa5e/coverage-7.14.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cfe5a5fec635799ef33428f1e5e61bafa45a92a96190ba731561ba558ccc214d", size = 246981, upload-time = "2026-05-26T20:38:19.266Z" }, - { url = "https://files.pythonhosted.org/packages/fc/10/1c7d04c13040dac531d21b712bbe08f902e6dd9b58f5d77875c4d030f8f2/coverage-7.14.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:62a9f70b52e0b5a95cfef4a5c5641b06983cadc5e538a3feeb5c00211f523ac2", size = 248812, upload-time = "2026-05-26T20:38:20.75Z" }, - { url = "https://files.pythonhosted.org/packages/c1/65/2a38a4607ef27cadcfbcee034dba5830ae2569f90144a0f4c7dbf47d30b0/coverage-7.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c18ebc343e15be53049b3a2dce38fe82d58f37e20ab9094b3a39c0aa4f6bb47", size = 250675, upload-time = "2026-05-26T20:38:22.159Z" }, - { url = "https://files.pythonhosted.org/packages/c9/a2/a446ed9752a4a59b79e0fb6cbb319f6facb2183045c0725462625e66f87e/coverage-7.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b84ffdf877644e7096aa936991efeed873f7f3df57b9cd001312b7668ab08550", size = 252590, upload-time = "2026-05-26T20:38:23.63Z" }, - { url = "https://files.pythonhosted.org/packages/9e/fd/e81fbd7ba752365546e9842b1cbdaad3d6919d2a522c590aef16a281ec5e/coverage-7.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e854312c4103f2ad4c0dc023b69b77ebfd2c89db5f86c4c94dc2353f9a92167e", size = 247691, upload-time = "2026-05-26T20:38:25.057Z" }, - { url = "https://files.pythonhosted.org/packages/53/35/f3c26fdaae9ea937d154ca4d372e5ea0a4167ff70d36c6074ac2eacb2f83/coverage-7.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c643734307300234fafa36bf2a040a7235f8f177ea1fd6ec1423aea6fb7b929f", size = 248716, upload-time = "2026-05-26T20:38:26.406Z" }, - { url = "https://files.pythonhosted.org/packages/2e/14/940b6c49551fd343e8507ee2b0ba7af5d0aa04ed5bf768285cb7c72a9884/coverage-7.14.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:84ac9499e48700399a5dd0ea7085b5091961fec52c68d66b4ec0d3cf7f4441b1", size = 246721, upload-time = "2026-05-26T20:38:28.282Z" }, - { url = "https://files.pythonhosted.org/packages/aa/2c/40fc0634186c28292a662dff578866b3913983d6c375a3c2a74020938719/coverage-7.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:7f02d09f70776579b926d889a4c9c235070a1f47c40458aeaca563fae5acfdb5", size = 250533, upload-time = "2026-05-26T20:38:29.753Z" }, - { url = "https://files.pythonhosted.org/packages/de/e3/2c26bf1e811f9df991ff2a9bdddebdd13ee0665d564df7d05979f9146297/coverage-7.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:ce66d8e46da2bb5ee313a745cbd2e391d319176c1f7a9451bfcd3a2fb920859b", size = 246990, upload-time = "2026-05-26T20:38:31.516Z" }, - { url = "https://files.pythonhosted.org/packages/a8/b0/060260ef56bd92363ebdce0c7095ce422b06e69aae71828efeca473ab1ca/coverage-7.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c912c259304cfb5ee584481cfb7ce1ff932b4d61e6c9140b8f19cb7b5ed82332", size = 247593, upload-time = "2026-05-26T20:38:33.065Z" }, - { url = "https://files.pythonhosted.org/packages/63/f3/501502046efeb0d6d94b5ca54941d95f1184183dd6bdb7f283985783bb4a/coverage-7.14.1-cp310-cp310-win32.whl", hash = "sha256:1238cb94638e610e972c60dac68e813f868dc7d6e982535270558443058d9d59", size = 222330, upload-time = "2026-05-26T20:38:35.36Z" }, - { url = "https://files.pythonhosted.org/packages/a0/5d/1bf99f2c558f128faf7906817ccbdb576ba815d3b41ce2ac1719b70a3663/coverage-7.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:fc459e5d73be2d6332fcfe8dbf3d8994671fe33c700f4565988ecfa511547253", size = 223261, upload-time = "2026-05-26T20:38:37.196Z" }, - { url = "https://files.pythonhosted.org/packages/7d/d7/477ad149490e6cb849f28abea1dabb9c823cea72e7500c81b4240ce619c0/coverage-7.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:478b5bcd63c2e1357c5c7e16c070690df7b07f676b1c114d7b93e533c664309f", size = 219848, upload-time = "2026-05-26T20:38:38.715Z" }, - { url = "https://files.pythonhosted.org/packages/91/82/a5eb47257c50601bb7b9a9d2857c67b7a3a85ad74180eb2c98bb1fbe0ce5/coverage-7.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a24a81f9715ee42ef59a316cc11611c98fe23920f7c81861315c9f3ff4a230f4", size = 220354, upload-time = "2026-05-26T20:38:40.232Z" }, - { url = "https://files.pythonhosted.org/packages/43/8b/78419b5391a5cb706b6544390507e469d83ffc9a8248b02c4011aceb9365/coverage-7.14.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:196a13319ad88d6d8ef5ab489ec4f44ddde2143c0c7d5b27786f6c3ffd56a7e1", size = 250771, upload-time = "2026-05-26T20:38:41.782Z" }, - { url = "https://files.pythonhosted.org/packages/77/63/e77aaacd491182210d639636b7a8bba23ffffa9b82aa3762da9431855fa9/coverage-7.14.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d452fd08b5c72c5167c93e6867b5c08500bd40f2a21e1e854a500550b6cc36f", size = 252683, upload-time = "2026-05-26T20:38:43.305Z" }, - { url = "https://files.pythonhosted.org/packages/65/1c/a022e3cfbec2ac241640003cb3a817e161d9c7f5aa9b49173756cdc03204/coverage-7.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23bf7fa51ac02e07fc7c96849b82946da47ae862dc8f86d183b2a4864fc38129", size = 254791, upload-time = "2026-05-26T20:38:45.361Z" }, - { url = "https://files.pythonhosted.org/packages/61/d6/967e408aca4c1ceb88cb0cc677169110ae7f5995fb5eaf5fb1f5a1bb8f5d/coverage-7.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bcaa50684dcaadfa599ac48f81103c756d791cfd85c97203d2217c593d48b860", size = 256748, upload-time = "2026-05-26T20:38:46.91Z" }, - { url = "https://files.pythonhosted.org/packages/b8/be/869188f7fe28638078ec479331ace6dc5f7b40b7153eb616f47ab79404d8/coverage-7.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4ea1c034f95c9b056e856b794630b17f9fa3d57e4800ff1e503d3be0f9c9078c", size = 250907, upload-time = "2026-05-26T20:38:48.493Z" }, - { url = "https://files.pythonhosted.org/packages/07/aa/adb7d3b4278d690e68703abcd76ab1b948242e3668d921711551b78f9ddb/coverage-7.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c7e057326434e441306226fbeb5d1aaf14a2637efe97ba668306635835f32ad7", size = 252483, upload-time = "2026-05-26T20:38:50.074Z" }, - { url = "https://files.pythonhosted.org/packages/43/61/331c74103c62dcb0c4b9b3a0de9a61aca016208b0a90f109592a9f9ecc28/coverage-7.14.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:59baf88468dbc8d63b1887afd92bda52e40bb1561696e5819670601403810cec", size = 250545, upload-time = "2026-05-26T20:38:51.613Z" }, - { url = "https://files.pythonhosted.org/packages/f6/b6/c5dae3c104d89be04828f61810e6b3473825482e4c288cc4ed04553e08ae/coverage-7.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:d34d75f892b3ab73ba11cab5442cce7b3e168fd64162b16f0e1e0d09c508edef", size = 254310, upload-time = "2026-05-26T20:38:53.503Z" }, - { url = "https://files.pythonhosted.org/packages/ad/a1/2b9d5863e3b83c01ad8199e3c597802fbb3a9dc90b058885804c20296d31/coverage-7.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3a56abc20a472baf0304c455721bc601477440d28ecfde8a03dde79ede07e0df", size = 250266, upload-time = "2026-05-26T20:38:55.414Z" }, - { url = "https://files.pythonhosted.org/packages/7f/5e/0e511fbdb269359be26fe678a1c3fa1f2aa2a01573cc3f54268c8d6d4797/coverage-7.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6a3cb83d1552c0cd1b4906655b6a33fd4a8473229633a901c6b73bf86914dee9", size = 251174, upload-time = "2026-05-26T20:38:57.141Z" }, - { url = "https://files.pythonhosted.org/packages/85/10/e55307b622b3dd9671cb321824502dc10f93e72f2802b9946159a8edadeb/coverage-7.14.1-cp311-cp311-win32.whl", hash = "sha256:10274a1fbeb8ec5d72966e17bb198a3104257aca4ac09d98667c5f8aca8c8548", size = 222354, upload-time = "2026-05-26T20:38:58.727Z" }, - { url = "https://files.pythonhosted.org/packages/71/cf/107421693cfb71e4f1ca5bf70443f64d4161878068d07a3e51c7ad21d17b/coverage-7.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:87ebdf787d4888e3f3f2d523eadc6e18c6d18c6d0eb173801a189641627fb37e", size = 223290, upload-time = "2026-05-26T20:39:00.413Z" }, - { url = "https://files.pythonhosted.org/packages/b8/1d/3e3644585eb29e9dafefb19555078529a4d7cce12bd21929664eea989277/coverage-7.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:dd34767fa19848d35659ffc0a75314f58c7af3f1cd87ec521e8292a1238398a3", size = 221953, upload-time = "2026-05-26T20:39:02.159Z" }, - { url = "https://files.pythonhosted.org/packages/3d/b7/bdbb725ba02c5b42825b200c940f38b7a54fcad24627b7192f78f8110d76/coverage-7.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a06c76364a9360e33d6d23769aefdf7f66f38e2ffb60ceb1baaa4989d83b695c", size = 220022, upload-time = "2026-05-26T20:39:03.702Z" }, - { url = "https://files.pythonhosted.org/packages/72/81/fdc0898a55c6219223291ec1a1fe89966ef212ce82276aa0899df84b5de0/coverage-7.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fad54e871165f6ec2f536063ac74c3104508a12963e64072ba44bd822de52b0c", size = 220379, upload-time = "2026-05-26T20:39:05.381Z" }, - { url = "https://files.pythonhosted.org/packages/de/72/de048c4a25e13bce59ac6a339351c10bdf2515e07459afcdaf04dc3143a2/coverage-7.14.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:84b535f00655ecafe1d929d1fb00ed5d6fa3051ea643ab2c161a3887b86f294b", size = 251888, upload-time = "2026-05-26T20:39:07.367Z" }, - { url = "https://files.pythonhosted.org/packages/28/30/300c343f68beb9d4cbb64ec81e58c5b6b80b56927f72d2b38654ac26e013/coverage-7.14.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6b6b0853b895fe0e98cbfc580d1ec3393d9302b4b1e96a77b3f5c91fdab899e6", size = 254624, upload-time = "2026-05-26T20:39:09.037Z" }, - { url = "https://files.pythonhosted.org/packages/b1/ed/7b25642496e8170b6bac14adce00537c6e5fa2d586159401a4de3e8b49e6/coverage-7.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:442cc9c952b2df400cda54bb04ab87330cf2cd08a8692cbbea36773531eb6f37", size = 255739, upload-time = "2026-05-26T20:39:10.889Z" }, - { url = "https://files.pythonhosted.org/packages/7f/a2/abd210b8c4e29c24e4624916db97bb519097a91034aaeb767f937e7da794/coverage-7.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8270544c361ed405a27a060dbc9ed2c124b084d96dfdc2d9a2510482aef981ad", size = 257998, upload-time = "2026-05-26T20:39:12.722Z" }, - { url = "https://files.pythonhosted.org/packages/7f/24/7c50beed3792fe62f6ce0545c6686ce83379719e2c0276179333d97eae92/coverage-7.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:48b283b1dd6372e8de2a7a9a4c4d5dc06f4d4fd209b876f3c88a7a205a0c8f84", size = 252296, upload-time = "2026-05-26T20:39:14.259Z" }, - { url = "https://files.pythonhosted.org/packages/15/05/0f874628ebcbfc77ead559ff210281ef06a97db08481832e7dd39274a135/coverage-7.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5b0c99ba93a07d56f6df340bb79be53202a082b2fdb81bfe6190b741a3470d54", size = 253658, upload-time = "2026-05-26T20:39:15.923Z" }, - { url = "https://files.pythonhosted.org/packages/99/6f/ca6ad067364b337ef997802115e7ecad2abd2248b05471464b0dea02b4d4/coverage-7.14.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e471bc5769ff073b058cfadb0d736b56ce067c8560eabeb0da88462df98c23e7", size = 251803, upload-time = "2026-05-26T20:39:17.537Z" }, - { url = "https://files.pythonhosted.org/packages/c0/30/b9b4d377cd9f40baf228068f5a81faf8450c6228503011bd499708483a50/coverage-7.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f497a1ea81d4cd7c10ddcaa685135b9aabd291af3d55775a9ddf3cb7a364cdd9", size = 255873, upload-time = "2026-05-26T20:39:19.414Z" }, - { url = "https://files.pythonhosted.org/packages/3c/21/7c721a9e5e6bb88547d30a787aefb97512d3f54c1324c7488d9b3743f7f9/coverage-7.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2222be86d0b54f5dd5a38f45f17f315f737245e857bf0bdedc70734f84a13c02", size = 251372, upload-time = "2026-05-26T20:39:21.169Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8c/f8ae5a2200130e1503cd7661a6cd3b2b7bacef98277fbf3571fb13f8b766/coverage-7.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:85e85586565842f6932abebd4c18bcb1074223dc0b3576e7d173ca710622813a", size = 253245, upload-time = "2026-05-26T20:39:23.097Z" }, - { url = "https://files.pythonhosted.org/packages/34/62/70a9024672a5f6910517d9628c52c9afbdd3cf8f46426af52bb148a56fff/coverage-7.14.1-cp312-cp312-win32.whl", hash = "sha256:4a28fd227808366b196a75476dced2eb35b351d6766ba9c858dc93319e87f4f1", size = 222567, upload-time = "2026-05-26T20:39:24.868Z" }, - { url = "https://files.pythonhosted.org/packages/f6/81/8b7cd386839b039ebe1855733b9f9449a8dec5d79564018234f185a7fa70/coverage-7.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:54acdb6674a4661768d7bf7db32dfb9f46ab1d764f8aba6df75ce1a6a088724e", size = 223372, upload-time = "2026-05-26T20:39:26.603Z" }, - { url = "https://files.pythonhosted.org/packages/ae/ba/b44d472022f620d289d95fa830143235c0c36461c6f2437ea8d51e5481ed/coverage-7.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:99cd41ff91afd94896fea3bc002706b6ae4ce95727d06e4a0f39c0a8d8bd8b1a", size = 221989, upload-time = "2026-05-26T20:39:28.242Z" }, - { url = "https://files.pythonhosted.org/packages/8a/9e/5f6d56327c62b185225d145191c607e07515294a0aa6338e58805cd4a5ac/coverage-7.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:be9f2c802dcfce3f71298303aa5dad0dce440a76c52f2f60dacd8656dab78793", size = 220044, upload-time = "2026-05-26T20:39:29.902Z" }, - { url = "https://files.pythonhosted.org/packages/75/92/e82aca356744cbbc0f77a0b623e38918c1872361963413a3bab5d0340393/coverage-7.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6223a72fd0e4c7156353ec0f08a5f93623e1d3034d0e2683b9bb8ea674131b1d", size = 220412, upload-time = "2026-05-26T20:39:31.561Z" }, - { url = "https://files.pythonhosted.org/packages/27/c9/385bde0bf7ed0f4bf3a7ee5367060a86b5d218718cfd6fb943c0f836b34f/coverage-7.14.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7279d2110a28cebc738b6459ecda2771735a4c18465fbbd36b3288fe5ed92247", size = 251412, upload-time = "2026-05-26T20:39:33.337Z" }, - { url = "https://files.pythonhosted.org/packages/51/8c/23faf6a2343a0d17f960a4bd56c43bc7eb4cf312f774dd6ceebd82c7d8fc/coverage-7.14.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9eeb3fcbc13ba40dfbdb22d01d196a28e9cef9ed4c29b60061a1e0e823a9929d", size = 254008, upload-time = "2026-05-26T20:39:35.009Z" }, - { url = "https://files.pythonhosted.org/packages/42/06/36f4aa9ca8a815e6036156e80706a67828bb97bd826948244f6996dda957/coverage-7.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f0cfc27c539f07cf5c0a4cfe211d0b6cae039f8f40526dbaa71944e64b50a7b", size = 255241, upload-time = "2026-05-26T20:39:36.71Z" }, - { url = "https://files.pythonhosted.org/packages/ca/79/95266316352f90f6b1c6736bb413302edfde2453fb32422d3911642691b3/coverage-7.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:221c70f316241a78e77e607c227cefc8808d4e08f28d99c04f35694690e940be", size = 257373, upload-time = "2026-05-26T20:39:38.412Z" }, - { url = "https://files.pythonhosted.org/packages/e3/9c/58316d1f66c488b5fca8a0eb3e98348807813efa8a0d0833b9021be27488/coverage-7.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:da028256b04ec30e5e0114b6f76172938c313991f0a2d3d894271315cf5d5e43", size = 251635, upload-time = "2026-05-26T20:39:40.268Z" }, - { url = "https://files.pythonhosted.org/packages/ef/5a/ca2398a568e16fed7bb713e84ba3603a7164fb65779abe645c565ec890d5/coverage-7.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76a085d7005236a767e3426148b2c407e53ad61695c562f8a81da2d373324901", size = 253373, upload-time = "2026-05-26T20:39:42.145Z" }, - { url = "https://files.pythonhosted.org/packages/6e/2c/0396562c32deaebe7be51d865b3a41e9a87d7561acafe1a28f53b07e019a/coverage-7.14.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b553d04b5e778a8e56d57eb134aff42a92718ecba45e79c4764ecfa40efd92ff", size = 251341, upload-time = "2026-05-26T20:39:43.907Z" }, - { url = "https://files.pythonhosted.org/packages/fd/8f/a94f9221184c9cae1ee115820e3798e48b6b17777a9f19e46fb9a0c8dc74/coverage-7.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:46f714d2fb8ae2f4f29f23ada7f1e79b759fff5a70f94a1dac23af204c3ec9e4", size = 255497, upload-time = "2026-05-26T20:39:46.166Z" }, - { url = "https://files.pythonhosted.org/packages/71/69/505d70e47db1eaebcd002c39759707621ef184cd6b1ae084d9f41293f323/coverage-7.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1896f5e19ff3f0431c7ce2172adc54890fd97f86b59ced8ca1649145d9ffe35d", size = 251159, upload-time = "2026-05-26T20:39:48.03Z" }, - { url = "https://files.pythonhosted.org/packages/e0/aa/58681c383aa33a9d2ed40a02d7a22fbf780d1fa4d575396365777828198c/coverage-7.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:62fd185ef9df3c33d1c8178c5af105f762afbad96038de9a4ae100aa6297ca33", size = 252934, upload-time = "2026-05-26T20:39:49.872Z" }, - { url = "https://files.pythonhosted.org/packages/eb/fd/11c928cd6bdffc7074bb5965c173d9ebf517fb00205e1da524b98d29ef92/coverage-7.14.1-cp313-cp313-win32.whl", hash = "sha256:ab4af6352741a604c431c6072fce5bee33bf0f20dc7a56618d6bf6bb89e9810c", size = 222584, upload-time = "2026-05-26T20:39:51.68Z" }, - { url = "https://files.pythonhosted.org/packages/6f/92/fb416fc26d340dcba19518c418d6048e913186e17243982c5e435e41fa7a/coverage-7.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:7af486dabe8954d03b087f0021540897afe084f04e16ff5579e08cc46f871416", size = 223394, upload-time = "2026-05-26T20:39:53.472Z" }, - { url = "https://files.pythonhosted.org/packages/73/c6/02d56e3867972f77d5036de924643f26c056e848f00452cafb4dbc3c29b4/coverage-7.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:2224f89ffd0c5605ccce1ed7a584da162bc7c55f601ab1c946bc9de31a486b42", size = 222015, upload-time = "2026-05-26T20:39:55.374Z" }, - { url = "https://files.pythonhosted.org/packages/4d/9e/fcc77914050df73f7662fa1f00902774c79c075a8388ab334074574bf77e/coverage-7.14.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:de286598cc65d2b489411174b1faec2f5a7775fb3201fd925db2a76b4030f37d", size = 220733, upload-time = "2026-05-26T20:39:57.189Z" }, - { url = "https://files.pythonhosted.org/packages/f7/67/2963cbdaf5cbadec44efa3a1e39eaa1f02df4079585f05387607a221e126/coverage-7.14.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:042c46ded7c288aeb07cf14a28b6c1e10b78fcba40171c3fa1e939377eeef0b5", size = 221086, upload-time = "2026-05-26T20:39:59.019Z" }, - { url = "https://files.pythonhosted.org/packages/c8/c5/8701645574e11881f2f47d8930f98bc48b5d43b25eb5b4430dfc4a2f9f48/coverage-7.14.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f4ddbe407477f04c45115d1a4e5bc480f753553b534d338d4c3358b1cdd0ea52", size = 262381, upload-time = "2026-05-26T20:40:00.822Z" }, - { url = "https://files.pythonhosted.org/packages/7c/28/7a64d73598263e0c5abd5084211a8474488d31b3c552ff531c719dfcff62/coverage-7.14.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d13e6725992e2d2fd7d81d4f5241952d13740121dfd501da09201be39b2c003a", size = 264458, upload-time = "2026-05-26T20:40:02.506Z" }, - { url = "https://files.pythonhosted.org/packages/fa/d8/4969179db9f7eb4df218e69540adf829d1c835f59452513d065d15446802/coverage-7.14.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f747dc8edcfe740130f28f32f3995e955494285717e86ee25af51db2219df08a", size = 266884, upload-time = "2026-05-26T20:40:04.421Z" }, - { url = "https://files.pythonhosted.org/packages/a6/78/a45d5794dbc9bafd97afc96a4377c86c7820d78b6cf51b89bc1d4e919275/coverage-7.14.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced2f09ef276fd58611a1ef502164ad266d2b75174e5a40cabbdb4033f9f6cf2", size = 268022, upload-time = "2026-05-26T20:40:06.298Z" }, - { url = "https://files.pythonhosted.org/packages/21/cb/4f5e354e9e3e67af96bd4e57113e6db6b22298c7168b13eec408a549903d/coverage-7.14.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b84800013769a78ccb9ef4659402e26d06867e337b61ec365f77ad008adea80e", size = 261631, upload-time = "2026-05-26T20:40:08.226Z" }, - { url = "https://files.pythonhosted.org/packages/ec/49/eced49af4cb996d5d8b7e94e736175c513e4facd3398507b89892b4326d8/coverage-7.14.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ea8cd6ca0ee9f616aaef3afc6882e32c2cbf18b00d96313ffd76af650574034d", size = 264443, upload-time = "2026-05-26T20:40:10.137Z" }, - { url = "https://files.pythonhosted.org/packages/f1/d8/5603a88a7c5913a6b54f6cb1a8c46f7b39cbb30f27cd3f492908da09b2d7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:aa5e304a873fabddc11e484e9b6b738bd38bd7bed17b09aa84eecf5332e8b8bb", size = 262069, upload-time = "2026-05-26T20:40:11.999Z" }, - { url = "https://files.pythonhosted.org/packages/f0/59/2ae3cb79da554a06c8619d6c88ea19dd1e4aed4b834b6a83bb1fa243bdc5/coverage-7.14.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:5a1c5215be81035e629d5bc756650634d0bf31991038db7a0eccb90f025ce16d", size = 265780, upload-time = "2026-05-26T20:40:13.858Z" }, - { url = "https://files.pythonhosted.org/packages/af/5f/b130c1dc999031f2648bd25317fbce505ad8d5562079b4ed81e736a84967/coverage-7.14.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:79058c47dae6788504b5effb319961bcd72d7240551464b91d474bc0ed186d69", size = 260970, upload-time = "2026-05-26T20:40:16.142Z" }, - { url = "https://files.pythonhosted.org/packages/87/d1/ec13ccddeb48ec963bdfa72a11224bac2584bd045ba13beca82f8113e9c7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:370c5afae3fa0658e11694a32b24c2778f6bc2d17718121f94ee185e69f26b54", size = 263157, upload-time = "2026-05-26T20:40:18.382Z" }, - { url = "https://files.pythonhosted.org/packages/cf/c2/cd91ead503045161092d3845f7bb95ea2f25131ce96d3e314dd835d91b9c/coverage-7.14.1-cp313-cp313t-win32.whl", hash = "sha256:3758dd0a7f1fa57365ef2e781df0f0731d38b6e3772259d13dae4bd8a958d4b1", size = 223259, upload-time = "2026-05-26T20:40:20.381Z" }, - { url = "https://files.pythonhosted.org/packages/71/9f/1e28d97e6bd2c76b07f38b7c02870f1371255ff6717f54eca578fcbbdd0e/coverage-7.14.1-cp313-cp313t-win_amd64.whl", hash = "sha256:6ff665fb023a77386fe11685190cee1f60a7d635994a30d9b0a061533d470fce", size = 224320, upload-time = "2026-05-26T20:40:22.316Z" }, - { url = "https://files.pythonhosted.org/packages/a9/e0/d936e908f0e1efa55e52b91e01b52f1055cef5e1ab2718493390ed8e2fb8/coverage-7.14.1-cp313-cp313t-win_arm64.whl", hash = "sha256:17a5a241e5997621a956a7f402a7433ef4221e5152809b785bec79e2323799f1", size = 222577, upload-time = "2026-05-26T20:40:24.894Z" }, - { url = "https://files.pythonhosted.org/packages/d6/34/fc2f101b151af3799a101f0550b0454aa008afdc0add677394ec4aa8ea10/coverage-7.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d5ed429d0b8edaac649e889b4ffcedb6c80b06629a3f93050e3dddfb99235bee", size = 220091, upload-time = "2026-05-26T20:40:27.249Z" }, - { url = "https://files.pythonhosted.org/packages/3d/a7/1ebae2ab5b961b5c79bb09fe7b3ac99edb190d8be4a8c510b2cf66f46468/coverage-7.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8011224a62280e50dab346960c03cf47aca1a1e09e608c0fb33fd6e0cc8e9500", size = 220421, upload-time = "2026-05-26T20:40:30.084Z" }, - { url = "https://files.pythonhosted.org/packages/5e/90/92aca9cf0acc95123c96cd1eb1f08917897a7f5dee01e15738922971ec31/coverage-7.14.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:12c42ec1e14f553c4f817e989365982e646e27211f10a0f717855b94a79c8906", size = 251466, upload-time = "2026-05-26T20:40:32.542Z" }, - { url = "https://files.pythonhosted.org/packages/26/2b/78048cbe3b999f6cbf9cc0d90abba6a88a3e0863a8c1c6cbc762f3f8802f/coverage-7.14.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:06144cd511cf2624873a035c5069cf297144f6e77a73ee3d7a55b605ec5efb42", size = 253973, upload-time = "2026-05-26T20:40:34.473Z" }, - { url = "https://files.pythonhosted.org/packages/8e/21/c2e33b29d1cfde484a19d437afc343c6cd30b08d78cbbf9f5aff14e57b2b/coverage-7.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a311d8e1da24be5c1ccf85cbfb06315dbaa1703d5a1eab3f6432c72b837917c8", size = 255318, upload-time = "2026-05-26T20:40:38.154Z" }, - { url = "https://files.pythonhosted.org/packages/8e/ee/aad2f108d63b769121005302f16bf66db8625c88ceaba466942e09a2607e/coverage-7.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c79cead5b5bc584d9c71451cb984d0e3a84e0c0937379c8efcbf27c8d661b851", size = 257633, upload-time = "2026-05-26T20:40:40.164Z" }, - { url = "https://files.pythonhosted.org/packages/c2/f8/11a2c29b4fd76d9849f81d0bb812ec0017a9396df3217214e38934a8c837/coverage-7.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dcbf65f1f66a26cdd88c35cf68fb4729c5d1cd2e88added72420541dfb212034", size = 251488, upload-time = "2026-05-26T20:40:42.631Z" }, - { url = "https://files.pythonhosted.org/packages/c9/b8/9a5820de4b8ac2b71d85e3b5fb49108d7469c665f0e2ad0dd7569023e305/coverage-7.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fd86572566fb40189a8260446158235159bc7a82dfbc87a3b39cf4fb57fcec1c", size = 253329, upload-time = "2026-05-26T20:40:45.208Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ff/f33e4823667e27548e8fd8df44217515303f9808d0ff29817db56f87d990/coverage-7.14.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:7771b601718fdde84832c3a434ca9bbf4ae9adbc49d84198b4110700c3c77c36", size = 251291, upload-time = "2026-05-26T20:40:47.502Z" }, - { url = "https://files.pythonhosted.org/packages/68/9b/489db0ebb209054766b90a9014a45f6d26eb724c02ec21311c3733b5a644/coverage-7.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:39b21e212c55af06fa375e3dbf90a8a8e38792f3a910c580066d23563830ddd5", size = 255564, upload-time = "2026-05-26T20:40:49.372Z" }, - { url = "https://files.pythonhosted.org/packages/27/b5/16bc2d4c2409b23c7737edb68c83bc89e345f378050549fe1d75ac7d34d5/coverage-7.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f2302660e32562a532b442480121aef8aa61a5bdb20b30bf0adab29f10a5a4b4", size = 251107, upload-time = "2026-05-26T20:40:51.677Z" }, - { url = "https://files.pythonhosted.org/packages/7d/0c/2629997469a00cd069d588a41c9dc887610f2775ae89d250c4791e65272a/coverage-7.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03a6f93c1ec3b7f2e77b5dbcc5573a2c21f12529a5c6bbe0f16f72303cc2fa4d", size = 252764, upload-time = "2026-05-26T20:40:54.267Z" }, - { url = "https://files.pythonhosted.org/packages/d2/ee/f78d63c8f079e0d7211c7e2401fa17e311514534ba61bae03e4b287ce4ab/coverage-7.14.1-cp314-cp314-win32.whl", hash = "sha256:8a3ce026d73290f42f08dafecbd82c193a74df280461fbf97300fec51fd133ee", size = 222837, upload-time = "2026-05-26T20:40:56.496Z" }, - { url = "https://files.pythonhosted.org/packages/dc/b9/be539854f93a70dfbeec69117f33ec70dc42ff0b65b5b07ab8d40d04228e/coverage-7.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:114c95ef29302423b87d159075805f4ab973254a2638a5d7d046c94887cc87d7", size = 223650, upload-time = "2026-05-26T20:40:58.351Z" }, - { url = "https://files.pythonhosted.org/packages/fe/9e/24e2842fef40f35ac82ba3a7719c8023d011bf3bf652d0675316a9d088a1/coverage-7.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:a07891c3f4805442b31b71e84ba3cf29ed1aa9a428284e06deeb4b23e5b46343", size = 222218, upload-time = "2026-05-26T20:41:00.321Z" }, - { url = "https://files.pythonhosted.org/packages/0a/1d/ac0a9df5fe31c1e8bdd658074905fc12844a05c1a7e3fdb8417e97c31e23/coverage-7.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1101a5ebb083aecb625ebb6209d4105b58f647b093cb2dc8122d7b33f743cfe1", size = 220822, upload-time = "2026-05-26T20:41:02.281Z" }, - { url = "https://files.pythonhosted.org/packages/32/cf/f964fd9aff20323f9f1a726c97135f8a76bcd87b92dad141a456a43f3c64/coverage-7.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:851b9e1e4e8a4608e77c79714b2e77c0970d2ed7202a05e92ae407817481887b", size = 221084, upload-time = "2026-05-26T20:41:04.593Z" }, - { url = "https://files.pythonhosted.org/packages/d8/5e/7e5ef2aba844de2b80d678619fcf0841b42e3f37f16411226f3fe4c1016f/coverage-7.14.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d5b89cdfb2ee051b71e8c3c70bd81a9eff81100f736a269136fe1a68efe00474", size = 262454, upload-time = "2026-05-26T20:41:06.641Z" }, - { url = "https://files.pythonhosted.org/packages/64/62/75809bded87015cc4935524218a2a8ed8dd1a8498bfed30a2f4f7a4b4d34/coverage-7.14.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0177614a0370f227888b4e436a7c55686d6a9f90eb1ade2b624ba685a1686e86", size = 264578, upload-time = "2026-05-26T20:41:08.556Z" }, - { url = "https://files.pythonhosted.org/packages/f3/42/d33392dc14633525012d2d504fa1a33b05538bf535f5c1d64675e5754b78/coverage-7.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d69af5dea2de76fc485a83032a630523f985198b7e25be901ec60181587b01e", size = 266981, upload-time = "2026-05-26T20:41:10.824Z" }, - { url = "https://files.pythonhosted.org/packages/2a/49/0157c4428c2aca7f1e09d5565930586fd5ae36f1655f08b0daa7cf1fcae1/coverage-7.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:35ab22d91de736e8966b980dc355cbcdd2c6dbbcfe275f9a2991bc8a91b3df65", size = 268112, upload-time = "2026-05-26T20:41:12.966Z" }, - { url = "https://files.pythonhosted.org/packages/96/26/86b9ce71f4092b1ed325ce1421698081df1286b833400b6836912834d6e0/coverage-7.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:357d4e32935c36588aaba057d734fa32428c360c9fc2e4442afbf1b646beee6e", size = 261558, upload-time = "2026-05-26T20:41:15Z" }, - { url = "https://files.pythonhosted.org/packages/20/4c/c311210c5472cf5401d8422b0d7812cdd520f24417673afabda6c323faca/coverage-7.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:51bd64741cc6fa065abd300ede1afe5a5291ece9c31da8b24884deda48bcc3f8", size = 264447, upload-time = "2026-05-26T20:41:17.369Z" }, - { url = "https://files.pythonhosted.org/packages/fb/71/59513f8710ed3e6b0ac0a050a5b7e977bb9c9e880354863b5d00d8809256/coverage-7.14.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:9132cd363a68a4c3daa7c8704a654b1e39d3360f6f5b8ddd470608a945236c07", size = 262048, upload-time = "2026-05-26T20:41:19.309Z" }, - { url = "https://files.pythonhosted.org/packages/84/8d/bceed32dc494f5bbf50f775cd2e78ca814953942b5ea28d3c1c3ac316f14/coverage-7.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:07c6290b1697b862c0478eab545eec949a0d0e4d6d03497f446d706da3b4f2de", size = 265781, upload-time = "2026-05-26T20:41:21.559Z" }, - { url = "https://files.pythonhosted.org/packages/e7/c5/9348fe40dbfd4991aaf78df2c6c3098bfb2cc834d1fd362a64b4efef855a/coverage-7.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5ea0c297e27133853b4d8a3eb799bff5a2dbd9f2f41537a240d337ac9b4df890", size = 260896, upload-time = "2026-05-26T20:41:23.428Z" }, - { url = "https://files.pythonhosted.org/packages/ca/92/1ea0f03929da7cf87206b1fa24f4c8e9c158be0455481af29ec0a1f3503f/coverage-7.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:01b7733daad0237daa01ef80fe2dfceffc911e6a17fa7b55d14aa8214eaaaecd", size = 263214, upload-time = "2026-05-26T20:41:25.419Z" }, - { url = "https://files.pythonhosted.org/packages/f6/a9/b2493c054c0e01a643266742ab45e15744e60743f9260cd930c7142b1124/coverage-7.14.1-cp314-cp314t-win32.whl", hash = "sha256:6adc5a36984624a70bf11d7184e20fa0a49aa7c47ffab43804106a1a695ea22e", size = 223624, upload-time = "2026-05-26T20:41:27.795Z" }, - { url = "https://files.pythonhosted.org/packages/fc/bd/3e1e6a57fccd2d7c83fcdf338e93ba98eb85c6e877dd34731ac585375490/coverage-7.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:ddf799247318f34dbcd2efa8c95a8d0642674e926bb1774cf9b63dfd2a389d1c", size = 224728, upload-time = "2026-05-26T20:41:30.098Z" }, - { url = "https://files.pythonhosted.org/packages/bb/d7/31066cf1d2f0c6c797fce911bcfa01dd35642dc6da992a950256097c5860/coverage-7.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:145986fe66647eb489f18d9a997567a3fd358584c4b5a808769113abc07466af", size = 222752, upload-time = "2026-05-26T20:41:32.123Z" }, - { url = "https://files.pythonhosted.org/packages/8a/3c/1a983b9a745d7f83d53f057bcc5bf79ba6a2bbc08266b3f0c7d6fe630c9b/coverage-7.14.1-py3-none-any.whl", hash = "sha256:a252f21c27e38347e60111a3266b03827422a7d5525951aceee313aa68bab1d2", size = 211815, upload-time = "2026-05-26T20:41:34.078Z" }, + { url = "https://files.pythonhosted.org/packages/69/33/e8c48488c29a73fd089f9d71f9653c1be7478f2ad6b5bc870db11a55d23d/coverage-7.13.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0723d2c96324561b9aa76fb982406e11d93cdb388a7a7da2b16e04719cf7ca5", size = 219255, upload-time = "2026-03-17T10:29:51.081Z" }, + { url = "https://files.pythonhosted.org/packages/da/bd/b0ebe9f677d7f4b74a3e115eec7ddd4bcf892074963a00d91e8b164a6386/coverage-7.13.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:52f444e86475992506b32d4e5ca55c24fc88d73bcbda0e9745095b28ef4dc0cf", size = 219772, upload-time = "2026-03-17T10:29:52.867Z" }, + { url = "https://files.pythonhosted.org/packages/48/cc/5cb9502f4e01972f54eedd48218bb203fe81e294be606a2bc93970208013/coverage-7.13.5-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:704de6328e3d612a8f6c07000a878ff38181ec3263d5a11da1db294fa6a9bdf8", size = 246532, upload-time = "2026-03-17T10:29:54.688Z" }, + { url = "https://files.pythonhosted.org/packages/7d/d8/3217636d86c7e7b12e126e4f30ef1581047da73140614523af7495ed5f2d/coverage-7.13.5-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a1a6d79a14e1ec1832cabc833898636ad5f3754a678ef8bb4908515208bf84f4", size = 248333, upload-time = "2026-03-17T10:29:56.221Z" }, + { url = "https://files.pythonhosted.org/packages/2b/30/2002ac6729ba2d4357438e2ed3c447ad8562866c8c63fc16f6dfc33afe56/coverage-7.13.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79060214983769c7ba3f0cee10b54c97609dca4d478fa1aa32b914480fd5738d", size = 250211, upload-time = "2026-03-17T10:29:57.938Z" }, + { url = "https://files.pythonhosted.org/packages/6c/85/552496626d6b9359eb0e2f86f920037c9cbfba09b24d914c6e1528155f7d/coverage-7.13.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:356e76b46783a98c2a2fe81ec79df4883a1e62895ea952968fb253c114e7f930", size = 252125, upload-time = "2026-03-17T10:29:59.388Z" }, + { url = "https://files.pythonhosted.org/packages/44/21/40256eabdcbccdb6acf6b381b3016a154399a75fe39d406f790ae84d1f3c/coverage-7.13.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0cef0cdec915d11254a7f549c1170afecce708d30610c6abdded1f74e581666d", size = 247219, upload-time = "2026-03-17T10:30:01.199Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e8/96e2a6c3f21a0ea77d7830b254a1542d0328acc8d7bdf6a284ba7e529f77/coverage-7.13.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dc022073d063b25a402454e5712ef9e007113e3a676b96c5f29b2bda29352f40", size = 248248, upload-time = "2026-03-17T10:30:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/da/ba/8477f549e554827da390ec659f3c38e4b6d95470f4daafc2d8ff94eaa9c2/coverage-7.13.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9b74db26dfea4f4e50d48a4602207cd1e78be33182bc9cbf22da94f332f99878", size = 246254, upload-time = "2026-03-17T10:30:04.832Z" }, + { url = "https://files.pythonhosted.org/packages/55/59/bc22aef0e6aa179d5b1b001e8b3654785e9adf27ef24c93dc4228ebd5d68/coverage-7.13.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ad146744ca4fd09b50c482650e3c1b1f4dfa1d4792e0a04a369c7f23336f0400", size = 250067, upload-time = "2026-03-17T10:30:06.535Z" }, + { url = "https://files.pythonhosted.org/packages/de/1b/c6a023a160806a5137dca53468fd97530d6acad24a22003b1578a9c2e429/coverage-7.13.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:c555b48be1853fe3997c11c4bd521cdd9a9612352de01fa4508f16ec341e6fe0", size = 246521, upload-time = "2026-03-17T10:30:08.486Z" }, + { url = "https://files.pythonhosted.org/packages/2d/3f/3532c85a55aa2f899fa17c186f831cfa1aa434d88ff792a709636f64130e/coverage-7.13.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7034b5c56a58ae5e85f23949d52c14aca2cfc6848a31764995b7de88f13a1ea0", size = 247126, upload-time = "2026-03-17T10:30:09.966Z" }, + { url = "https://files.pythonhosted.org/packages/aa/2e/b9d56af4a24ef45dfbcda88e06870cb7d57b2b0bfa3a888d79b4c8debd76/coverage-7.13.5-cp310-cp310-win32.whl", hash = "sha256:eb7fdf1ef130660e7415e0253a01a7d5a88c9c4d158bcf75cbbd922fd65a5b58", size = 221860, upload-time = "2026-03-17T10:30:11.393Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cc/d938417e7a4d7f0433ad4edee8bb2acdc60dc7ac5af19e2a07a048ecbee3/coverage-7.13.5-cp310-cp310-win_amd64.whl", hash = "sha256:3e1bb5f6c78feeb1be3475789b14a0f0a5b47d505bfc7267126ccbd50289999e", size = 222788, upload-time = "2026-03-17T10:30:12.886Z" }, + { url = "https://files.pythonhosted.org/packages/4b/37/d24c8f8220ff07b839b2c043ea4903a33b0f455abe673ae3c03bbdb7f212/coverage-7.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66a80c616f80181f4d643b0f9e709d97bcea413ecd9631e1dedc7401c8e6695d", size = 219381, upload-time = "2026-03-17T10:30:14.68Z" }, + { url = "https://files.pythonhosted.org/packages/35/8b/cd129b0ca4afe886a6ce9d183c44d8301acbd4ef248622e7c49a23145605/coverage-7.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:145ede53ccbafb297c1c9287f788d1bc3efd6c900da23bf6931b09eafc931587", size = 219880, upload-time = "2026-03-17T10:30:16.231Z" }, + { url = "https://files.pythonhosted.org/packages/55/2f/e0e5b237bffdb5d6c530ce87cc1d413a5b7d7dfd60fb067ad6d254c35c76/coverage-7.13.5-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0672854dc733c342fa3e957e0605256d2bf5934feeac328da9e0b5449634a642", size = 250303, upload-time = "2026-03-17T10:30:17.748Z" }, + { url = "https://files.pythonhosted.org/packages/92/be/b1afb692be85b947f3401375851484496134c5554e67e822c35f28bf2fbc/coverage-7.13.5-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ec10e2a42b41c923c2209b846126c6582db5e43a33157e9870ba9fb70dc7854b", size = 252218, upload-time = "2026-03-17T10:30:19.804Z" }, + { url = "https://files.pythonhosted.org/packages/da/69/2f47bb6fa1b8d1e3e5d0c4be8ccb4313c63d742476a619418f85740d597b/coverage-7.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be3d4bbad9d4b037791794ddeedd7d64a56f5933a2c1373e18e9e568b9141686", size = 254326, upload-time = "2026-03-17T10:30:21.321Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d0/79db81da58965bd29dabc8f4ad2a2af70611a57cba9d1ec006f072f30a54/coverage-7.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d2afbc5cc54d286bfb54541aa50b64cdb07a718227168c87b9e2fb8f25e1743", size = 256267, upload-time = "2026-03-17T10:30:23.094Z" }, + { url = "https://files.pythonhosted.org/packages/e5/32/d0d7cc8168f91ddab44c0ce4806b969df5f5fdfdbb568eaca2dbc2a04936/coverage-7.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3ad050321264c49c2fa67bb599100456fc51d004b82534f379d16445da40fb75", size = 250430, upload-time = "2026-03-17T10:30:25.311Z" }, + { url = "https://files.pythonhosted.org/packages/4d/06/a055311d891ddbe231cd69fdd20ea4be6e3603ffebddf8704b8ca8e10a3c/coverage-7.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7300c8a6d13335b29bb76d7651c66af6bd8658517c43499f110ddc6717bfc209", size = 252017, upload-time = "2026-03-17T10:30:27.284Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f6/d0fd2d21e29a657b5f77a2fe7082e1568158340dceb941954f776dce1b7b/coverage-7.13.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:eb07647a5738b89baab047f14edd18ded523de60f3b30e75c2acc826f79c839a", size = 250080, upload-time = "2026-03-17T10:30:29.481Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ab/0d7fb2efc2e9a5eb7ddcc6e722f834a69b454b7e6e5888c3a8567ecffb31/coverage-7.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9adb6688e3b53adffefd4a52d72cbd8b02602bfb8f74dcd862337182fd4d1a4e", size = 253843, upload-time = "2026-03-17T10:30:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/ba/6f/7467b917bbf5408610178f62a49c0ed4377bb16c1657f689cc61470da8ce/coverage-7.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7c8d4bc913dd70b93488d6c496c77f3aff5ea99a07e36a18f865bca55adef8bd", size = 249802, upload-time = "2026-03-17T10:30:33.358Z" }, + { url = "https://files.pythonhosted.org/packages/75/2c/1172fb689df92135f5bfbbd69fc83017a76d24ea2e2f3a1154007e2fb9f8/coverage-7.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0e3c426ffc4cd952f54ee9ffbdd10345709ecc78a3ecfd796a57236bfad0b9b8", size = 250707, upload-time = "2026-03-17T10:30:35.2Z" }, + { url = "https://files.pythonhosted.org/packages/67/21/9ac389377380a07884e3b48ba7a620fcd9dbfaf1d40565facdc6b36ec9ef/coverage-7.13.5-cp311-cp311-win32.whl", hash = "sha256:259b69bb83ad9894c4b25be2528139eecba9a82646ebdda2d9db1ba28424a6bf", size = 221880, upload-time = "2026-03-17T10:30:36.775Z" }, + { url = "https://files.pythonhosted.org/packages/af/7f/4cd8a92531253f9d7c1bbecd9fa1b472907fb54446ca768c59b531248dc5/coverage-7.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:258354455f4e86e3e9d0d17571d522e13b4e1e19bf0f8596bcf9476d61e7d8a9", size = 222816, upload-time = "2026-03-17T10:30:38.891Z" }, + { url = "https://files.pythonhosted.org/packages/12/a6/1d3f6155fb0010ca68eba7fe48ca6c9da7385058b77a95848710ecf189b1/coverage-7.13.5-cp311-cp311-win_arm64.whl", hash = "sha256:bff95879c33ec8da99fc9b6fe345ddb5be6414b41d6d1ad1c8f188d26f36e028", size = 221483, upload-time = "2026-03-17T10:30:40.463Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01", size = 219554, upload-time = "2026-03-17T10:30:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422", size = 219908, upload-time = "2026-03-17T10:30:43.906Z" }, + { url = "https://files.pythonhosted.org/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f", size = 251419, upload-time = "2026-03-17T10:30:45.545Z" }, + { url = "https://files.pythonhosted.org/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5", size = 254159, upload-time = "2026-03-17T10:30:47.204Z" }, + { url = "https://files.pythonhosted.org/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376", size = 255270, upload-time = "2026-03-17T10:30:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256", size = 257538, upload-time = "2026-03-17T10:30:50.77Z" }, + { url = "https://files.pythonhosted.org/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c", size = 251821, upload-time = "2026-03-17T10:30:52.5Z" }, + { url = "https://files.pythonhosted.org/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5", size = 253191, upload-time = "2026-03-17T10:30:54.543Z" }, + { url = "https://files.pythonhosted.org/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09", size = 251337, upload-time = "2026-03-17T10:30:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9", size = 255404, upload-time = "2026-03-17T10:30:58.427Z" }, + { url = "https://files.pythonhosted.org/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf", size = 250903, upload-time = "2026-03-17T10:31:00.093Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c", size = 252780, upload-time = "2026-03-17T10:31:01.916Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf", size = 222093, upload-time = "2026-03-17T10:31:03.642Z" }, + { url = "https://files.pythonhosted.org/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810", size = 222900, upload-time = "2026-03-17T10:31:05.651Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de", size = 221515, upload-time = "2026-03-17T10:31:07.293Z" }, + { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", size = 219576, upload-time = "2026-03-17T10:31:09.045Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", size = 219942, upload-time = "2026-03-17T10:31:10.708Z" }, + { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", size = 250935, upload-time = "2026-03-17T10:31:12.392Z" }, + { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", size = 253541, upload-time = "2026-03-17T10:31:14.247Z" }, + { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", size = 254780, upload-time = "2026-03-17T10:31:16.193Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", size = 256912, upload-time = "2026-03-17T10:31:17.89Z" }, + { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", size = 251165, upload-time = "2026-03-17T10:31:19.605Z" }, + { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", size = 252908, upload-time = "2026-03-17T10:31:21.312Z" }, + { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", size = 250873, upload-time = "2026-03-17T10:31:23.565Z" }, + { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", size = 255030, upload-time = "2026-03-17T10:31:25.58Z" }, + { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", size = 250694, upload-time = "2026-03-17T10:31:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", size = 252469, upload-time = "2026-03-17T10:31:29.472Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", size = 222112, upload-time = "2026-03-17T10:31:31.526Z" }, + { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", size = 222923, upload-time = "2026-03-17T10:31:33.633Z" }, + { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", size = 221540, upload-time = "2026-03-17T10:31:35.445Z" }, + { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", size = 220262, upload-time = "2026-03-17T10:31:37.184Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", size = 220617, upload-time = "2026-03-17T10:31:39.245Z" }, + { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", size = 261912, upload-time = "2026-03-17T10:31:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", size = 263987, upload-time = "2026-03-17T10:31:43.724Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", size = 266416, upload-time = "2026-03-17T10:31:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", size = 267558, upload-time = "2026-03-17T10:31:48.293Z" }, + { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", size = 261163, upload-time = "2026-03-17T10:31:50.125Z" }, + { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", size = 263981, upload-time = "2026-03-17T10:31:51.961Z" }, + { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", size = 261604, upload-time = "2026-03-17T10:31:53.872Z" }, + { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", size = 265321, upload-time = "2026-03-17T10:31:55.997Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", size = 260502, upload-time = "2026-03-17T10:31:58.308Z" }, + { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", size = 262688, upload-time = "2026-03-17T10:32:00.141Z" }, + { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", size = 222788, upload-time = "2026-03-17T10:32:02.246Z" }, + { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", size = 223851, upload-time = "2026-03-17T10:32:04.416Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", size = 222104, upload-time = "2026-03-17T10:32:06.65Z" }, + { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", size = 219621, upload-time = "2026-03-17T10:32:08.589Z" }, + { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", size = 219953, upload-time = "2026-03-17T10:32:10.507Z" }, + { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", size = 250992, upload-time = "2026-03-17T10:32:12.41Z" }, + { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", size = 253503, upload-time = "2026-03-17T10:32:14.449Z" }, + { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", size = 254852, upload-time = "2026-03-17T10:32:16.56Z" }, + { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", size = 257161, upload-time = "2026-03-17T10:32:19.004Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", size = 251021, upload-time = "2026-03-17T10:32:21.344Z" }, + { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", size = 252858, upload-time = "2026-03-17T10:32:23.506Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", size = 250823, upload-time = "2026-03-17T10:32:25.516Z" }, + { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", size = 255099, upload-time = "2026-03-17T10:32:27.944Z" }, + { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", size = 250638, upload-time = "2026-03-17T10:32:29.914Z" }, + { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", size = 252295, upload-time = "2026-03-17T10:32:31.981Z" }, + { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", size = 222360, upload-time = "2026-03-17T10:32:34.233Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", size = 223174, upload-time = "2026-03-17T10:32:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", size = 221739, upload-time = "2026-03-17T10:32:38.736Z" }, + { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", size = 220351, upload-time = "2026-03-17T10:32:41.196Z" }, + { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", size = 220612, upload-time = "2026-03-17T10:32:43.204Z" }, + { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", size = 261985, upload-time = "2026-03-17T10:32:45.514Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", size = 264107, upload-time = "2026-03-17T10:32:47.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", size = 266513, upload-time = "2026-03-17T10:32:50.1Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", size = 267650, upload-time = "2026-03-17T10:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", size = 261089, upload-time = "2026-03-17T10:32:54.544Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", size = 263982, upload-time = "2026-03-17T10:32:56.803Z" }, + { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", size = 261579, upload-time = "2026-03-17T10:32:59.466Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", size = 265316, upload-time = "2026-03-17T10:33:01.847Z" }, + { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", size = 260427, upload-time = "2026-03-17T10:33:03.945Z" }, + { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", size = 262745, upload-time = "2026-03-17T10:33:06.285Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", size = 223146, upload-time = "2026-03-17T10:33:08.756Z" }, + { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", size = 224254, upload-time = "2026-03-17T10:33:11.174Z" }, + { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", size = 222276, upload-time = "2026-03-17T10:33:13.466Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, ] [package.optional-dependencies] @@ -333,15 +179,14 @@ toml = [ [[package]] name = "deepdiff" -version = "9.1.0" +version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cachebox" }, { name = "orderly-set" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f9/6b/6a4a5aaf38535eb332c2856aa08e73ed7c549d0851b1215401af0a2db1a7/deepdiff-9.1.0.tar.gz", hash = "sha256:07e9e366fab4297755153c4eab795ad4ef3cbd0d51660e847f5751c6bd727687", size = 382149, upload-time = "2026-05-15T20:18:05.751Z" } +sdist = { url = "https://files.pythonhosted.org/packages/24/20/63dd34163ed07393968128dc8c7ab948c96e47c4ce76976ea533de64909d/deepdiff-9.0.0.tar.gz", hash = "sha256:4872005306237b5b50829803feff58a1dfd20b2b357a55de22e7ded65b2008a7", size = 151952, upload-time = "2026-03-30T05:52:23.769Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/26/4a2bad8eb430d8d805a4642c4bff25103a37548d74ab346f8b1e024abcc5/deepdiff-9.1.0-py3-none-any.whl", hash = "sha256:80c0460e1993b04f6f0ca79abf25548b129fd218478c4ebb08f80560f5d10610", size = 184662, upload-time = "2026-05-15T20:18:03.956Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c4/da7089cd7aa4ab554f56e18a7fb08dcfed8fd2ae91fa528f5b1be207a148/deepdiff-9.0.0-py3-none-any.whl", hash = "sha256:b1ae0dd86290d86a03de5fbee728fde43095c1472ae4974bdab23ab4656305bd", size = 170540, upload-time = "2026-03-30T05:52:22.008Z" }, ] [[package]] @@ -371,7 +216,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -380,11 +225,11 @@ wheels = [ [[package]] name = "idna" -version = "3.18" +version = "3.13" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/cc/762dfb036166873f0059f3b7de4565e1b5bc3d6f28a414c13da27e442f99/idna-3.13.tar.gz", hash = "sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242", size = 194210, upload-time = "2026-04-22T16:42:42.314Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, + { url = "https://files.pythonhosted.org/packages/5d/13/ad7d7ca3808a898b4612b6fe93cde56b53f3034dcde235acb1f0e1df24c6/idna-3.13-py3-none-any.whl", hash = "sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3", size = 68629, upload-time = "2026-04-22T16:42:40.909Z" }, ] [[package]] @@ -419,99 +264,99 @@ wheels = [ [[package]] name = "librt" -version = "0.11.0" +version = "0.9.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/40/08/9e7f6b5d2b5bed6ad055cdd5925f192bb403a51280f86b56554d9d0699a2/librt-0.11.0.tar.gz", hash = "sha256:075dc3ef4458a278e0195cbf6ac9d38808d9b906c5a6c7f7f79c3888276a3fb1", size = 200139, upload-time = "2026-05-10T18:17:25.138Z" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/6b/3d5c13fb3e3c4f43206c8f9dfed13778c2ed4f000bacaa0b7ce3c402a265/librt-0.9.0.tar.gz", hash = "sha256:a0951822531e7aee6e0dfb556b30d5ee36bbe234faf60c20a16c01be3530869d", size = 184368, upload-time = "2026-04-09T16:06:26.173Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/83/10/37fd9e9ba96cb0bd742dfb20fc3d082e54bdbec759d7300df927f360ef07/librt-0.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6e94ebfcfa2d5e9926d6c3b9aa4617ffc42a845b4321fb84021b872358c82a0f", size = 141706, upload-time = "2026-05-10T18:15:16.129Z" }, - { url = "https://files.pythonhosted.org/packages/cf/72/1b1466f358e4a0b728051f69bc27e67b432c6eaa2e05b88db49d3785ae0d/librt-0.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ae627397a2f351560440d872d6f7c8dbb4072e57868e7b2fc5b8b430fe489d45", size = 142605, upload-time = "2026-05-10T18:15:18.148Z" }, - { url = "https://files.pythonhosted.org/packages/ca/85/ed26dd2f6bc9a0baf48306433e579e8d354d70b2bcb78134ed950a5d0e1e/librt-0.11.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc329359321b67d24efdf4bc69012b0597001649544db662c001db5a0184794c", size = 476555, upload-time = "2026-05-10T18:15:19.569Z" }, - { url = "https://files.pythonhosted.org/packages/66/fe/11891191c0e0a3fd617724e891f6e67a71a7658974a892b9a9a97fdb2977/librt-0.11.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:7e82e642ab0f7608ce2fe53d76ca2280a9ee33a1b06556142c7c6fe80a86fc33", size = 468434, upload-time = "2026-05-10T18:15:20.87Z" }, - { url = "https://files.pythonhosted.org/packages/6f/50/5ec949d7f9ce1a07af903aa3e13abb98b717923bdead6e719b2f824ccc07/librt-0.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:88145c15c67731d54283d135b03244028c750cc9edc334a96a4f5950ebdb2884", size = 496918, upload-time = "2026-05-10T18:15:22.616Z" }, - { url = "https://files.pythonhosted.org/packages/ea/c4/177336c7524e34875a38bf668e88b193a6723a4eb4045d07f74df6e1506c/librt-0.11.0-cp310-cp310-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9d36a51b3d93320b686588e27123f4995804dbf1bce81df78c02fc3c6eea9280", size = 490334, upload-time = "2026-05-10T18:15:24.2Z" }, - { url = "https://files.pythonhosted.org/packages/13/1f/da3112f7569eda3b49f9a2629bae1fe059812b6085df16c885f6454dff49/librt-0.11.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d00f3ac06a2a8b246327f11e186a53a100a4d5c7ed52346367e5ec751d51586c", size = 511287, upload-time = "2026-05-10T18:15:26.226Z" }, - { url = "https://files.pythonhosted.org/packages/fa/94/03fec301522e172d105581431223be56b27594ff46440ebfbb658a3735d5/librt-0.11.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:461bbceede621f1ffb8839755f8663e886087ee7af16294cab7fb4d782c62eeb", size = 517202, upload-time = "2026-05-10T18:15:27.965Z" }, - { url = "https://files.pythonhosted.org/packages/b7/6e/339f6e5a7b413ce014f1917a756dae630fe59cc99f34153205b1cb540901/librt-0.11.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0cad8a4d6a8ff03c9b76f9414caccd78e7cfbc8a2e12fa334d8e1d9932753783", size = 497517, upload-time = "2026-05-10T18:15:29.614Z" }, - { url = "https://files.pythonhosted.org/packages/cd/43/acdd5ce317cb46e8253ca9bfbdb8b12e68a24d745949336a7f3d5fb79ba0/librt-0.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f37aa505b3cf60701562eddb32df74b12a9e380c207fd8b06dd157a943ac7ea0", size = 538878, upload-time = "2026-05-10T18:15:30.928Z" }, - { url = "https://files.pythonhosted.org/packages/29/b5/7a25bb12e3172839f647f196b3e988318b7bb1ca7501732a225c4dce2ec0/librt-0.11.0-cp310-cp310-win32.whl", hash = "sha256:94663a21534637f0e787ec2a2a756022df6e5b7b2335a5cdd7d8e33d68a2af89", size = 100070, upload-time = "2026-05-10T18:15:32.551Z" }, - { url = "https://files.pythonhosted.org/packages/c6/0d/ebbcf4d77999c02c937b05d2b90ff4cd4dcc7e9a365ba132329ac1fe7a0f/librt-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:dec7db73758c2b54953fd8b7fe348c45188fe26b39ee18446196edd08453a5d4", size = 117918, upload-time = "2026-05-10T18:15:33.678Z" }, - { url = "https://files.pythonhosted.org/packages/fe/87/2bf31fe17587b29e3f93ec31421e2b1e1c3e349b8bf6c7c313dbad1d5340/librt-0.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:93d95bd45b7d58343d8b90d904450a545144eec19a002511163426f8ab1fae29", size = 141092, upload-time = "2026-05-10T18:15:34.795Z" }, - { url = "https://files.pythonhosted.org/packages/cf/08/5c5bf772920b7ebac6e32bc91a643e0ab3870199c0b542356d3baa83970a/librt-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ee278c769a713638cdacd4c0436d72156e75df3ebc0166ab2b9dc43acc386c9", size = 142035, upload-time = "2026-05-10T18:15:36.242Z" }, - { url = "https://files.pythonhosted.org/packages/06/20/662a03d254e5b000d838e8b345d83303ddb768c080fd488e40634c0fa66b/librt-0.11.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f230cb1cbc9faaa616f9a678f530ebcf186e414b6bcbd88b960e4ba1b92428d5", size = 475022, upload-time = "2026-05-10T18:15:37.56Z" }, - { url = "https://files.pythonhosted.org/packages/de/f3/aa81523e45184c6ec23dc7f63263362ec55f80a09d424c012359ecbe7e35/librt-0.11.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5d63c855d86938d9de93e265c9bd8c705b51ec494de5738340ee93767a686e4b", size = 467273, upload-time = "2026-05-10T18:15:39.182Z" }, - { url = "https://files.pythonhosted.org/packages/6b/6f/59c74b560ca8853834d5501d589c8a2519f4184f273a085ffd0f37a1cc47/librt-0.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:993f028be9e96a08d31df3479ac80d99be374d17f3b78e4796b3fd3c913d4e89", size = 497083, upload-time = "2026-05-10T18:15:40.634Z" }, - { url = "https://files.pythonhosted.org/packages/fe/7b/5aa4d2c9600a719401160bf7055417df0b2a47439b9d88286ce45e56b65f/librt-0.11.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:258d73a0aa66a055e65b2e4d1b8cdb23b9d132c5bb915d9547d804fcaed116cc", size = 489139, upload-time = "2026-05-10T18:15:41.934Z" }, - { url = "https://files.pythonhosted.org/packages/d6/31/9143803d7da6856a69153785768c4936864430eec0fd9461c3ea527d9922/librt-0.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0827efe7854718f04aaddf6496e96960a956e676fe1d0f04eb41511fd8ad06d5", size = 508442, upload-time = "2026-05-10T18:15:43.206Z" }, - { url = "https://files.pythonhosted.org/packages/2f/5a/bce08184488426bda4ccc2c4964ac048c8f68ae89bd7120082eef4233cfd/librt-0.11.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7753e57d6e12d019c0d8786f1c09c709f4c3fcc57c3887b24e36e6c06ec938b7", size = 514230, upload-time = "2026-05-10T18:15:44.761Z" }, - { url = "https://files.pythonhosted.org/packages/89/8c/bb5e213d254b7505a0e658da199d8ab719086632ce09eef311ab27976523/librt-0.11.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:11bd19822431cc21af9f27374e7ae2e58103c7d98bda823536a6c47f6bb2bb3d", size = 494231, upload-time = "2026-05-10T18:15:46.308Z" }, - { url = "https://files.pythonhosted.org/packages/9d/fb/541cdad5b1ab1300398c74c4c9a497b88e5074c21b1244c8f49731d3a284/librt-0.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:22bdf239b219d3993761a148ffa134b19e52e9989c84f845d5d7b71d70a17412", size = 537585, upload-time = "2026-05-10T18:15:47.629Z" }, - { url = "https://files.pythonhosted.org/packages/8f/f2/464bb69295c320cb06bddb4f14a4ec67934ee14b2bffb12b19fb7ab287ba/librt-0.11.0-cp311-cp311-win32.whl", hash = "sha256:46c60b61e308eb535fbd6fa622b1ee1bb2815691c1ad9c98bf7b84952ec3bc8d", size = 100509, upload-time = "2026-05-10T18:15:49.157Z" }, - { url = "https://files.pythonhosted.org/packages/6d/e7/a17ee1788f9e4fbf548c19f4afa07c92089b9e24fef6cb2410863781ef4c/librt-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:902e546ff044f579ff1c953ff5fce97b636fe9e3943996b2177710c6ef076f73", size = 118628, upload-time = "2026-05-10T18:15:50.345Z" }, - { url = "https://files.pythonhosted.org/packages/cc/c7/6c766214f9f9903bcfcfbef97d807af8d8f5aa3502d247858ab17582d212/librt-0.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:65ac3bc20f78aa0ee5ae84baa68917f89fef4af63e941084dd019a0d0e749f0c", size = 103122, upload-time = "2026-05-10T18:15:52.068Z" }, - { url = "https://files.pythonhosted.org/packages/8b/d0/07c77e067f0838949b43bd89232c29d72efebb9d2801a9750184eb706b71/librt-0.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b87504f1690a23b9a2cca841191a04f83895d4fc2dd04df91d82b1a04ca2ad46", size = 144147, upload-time = "2026-05-10T18:15:53.227Z" }, - { url = "https://files.pythonhosted.org/packages/7a/24/8493538fa4f62f982686398a5b8f68008138a75086abdea19ade64bf4255/librt-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40071fc5fe0ce8daa6de616702314a01e1250711682b0523d6ab8d4525910cb3", size = 143614, upload-time = "2026-05-10T18:15:54.657Z" }, - { url = "https://files.pythonhosted.org/packages/ff/1e/f8bad050810d9171f34a1648ed910e56814c2ba61639f2bd53c6377ae24b/librt-0.11.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:137e79445c896a0ea7b265f52d23954e05b64222ee1af69e2cb34219067cbb67", size = 485538, upload-time = "2026-05-10T18:15:56.117Z" }, - { url = "https://files.pythonhosted.org/packages/c0/fe/3594ebfbaf03084ba4b120c9ba5c3183fd938a48725e9bbe6ff0a5159ad8/librt-0.11.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:cca6644054e78746d8d4ef238681f9c34ff8b584fe6b988ecebb8db3b15e622a", size = 479623, upload-time = "2026-05-10T18:15:57.544Z" }, - { url = "https://files.pythonhosted.org/packages/b0/da/5d1876984b3746c85dbd219dbfcb73c85f54ee263fd32e5b2a632ec14571/librt-0.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5b0eea49f5562861ee8d757a32ef7d559c1d35be2aaaa1ec28941d74c9ffc8a", size = 513082, upload-time = "2026-05-10T18:15:58.805Z" }, - { url = "https://files.pythonhosted.org/packages/19/6e/55bdf5d5ca00c3e18430690bf2c953d8d3ffd3c337418173d33dec985dc9/librt-0.11.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0d1029d7e1ae1a7e647ed6fb5df8c4ce2dffefb7a9f5fd1376a4554d96dac09f", size = 508105, upload-time = "2026-05-10T18:16:00.2Z" }, - { url = "https://files.pythonhosted.org/packages/07/10/f1f23a7c595ee90ece4d35c851e5d104b1311a887ed1b4ac4c35bbd13da8/librt-0.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bc3ce6b33c5828d9e80592011a5c584cb2ce86edbc4088405f70da47dc1d1b3b", size = 522268, upload-time = "2026-05-10T18:16:01.708Z" }, - { url = "https://files.pythonhosted.org/packages/b6/02/5720f5697a7f54b78b3aefbe20df3a48cedcff1276618c4aa481177942ed/librt-0.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:936c5995f3514a42111f20099397d8177c79b4d7e70961e396c6f5a0a3566766", size = 527348, upload-time = "2026-05-10T18:16:03.496Z" }, - { url = "https://files.pythonhosted.org/packages/50/db/b4a47c6f91db4ff76348a0b3dd0cc65e090a078b765a810a62ff9434c3d3/librt-0.11.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9bc0ca6ad9381cbe8e4aa6e5726e4c80c78115a6e9723c599ed1d73e092bc49d", size = 516294, upload-time = "2026-05-10T18:16:05.173Z" }, - { url = "https://files.pythonhosted.org/packages/9e/58/9384b2f4eb1ed1d273d40948a7c5c4b2360213b402ef3be4641c06299f9c/librt-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:070aa8c26c0a74774317a72df8851facc7f0f012a5b406557ac56992d92e1ec8", size = 553608, upload-time = "2026-05-10T18:16:06.839Z" }, - { url = "https://files.pythonhosted.org/packages/21/7b/5aa8848a7c6a9278c79375146da1812e695754ceec5f005e6043461a7315/librt-0.11.0-cp312-cp312-win32.whl", hash = "sha256:6bf14feb84b05ae945277395451998c89c54d0def4070eb5c08de544930b245a", size = 101879, upload-time = "2026-05-10T18:16:08.103Z" }, - { url = "https://files.pythonhosted.org/packages/37/33/8a745436944947575b584231750a41417de1a38cf6a2e9251d1065651c09/librt-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:75672f0bc524ede266287d532d7923dbce94c7514ad07627bac3d0c6d92cc4d9", size = 119831, upload-time = "2026-05-10T18:16:09.174Z" }, - { url = "https://files.pythonhosted.org/packages/59/67/a6739ac96e28b7855808bdb0370e250606104a859750d209e5a0716fe7ab/librt-0.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:2f10cf143e4a9bb0f4f5af568a00df94a2d69ef41c2579584454bb0fe5cc642c", size = 103470, upload-time = "2026-05-10T18:16:10.369Z" }, - { url = "https://files.pythonhosted.org/packages/82/61/e59168d4d0bf2bf90f4f0caf7a001bfc60254c3af4586013b04dc3ef517b/librt-0.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:78dc31f7fdfe9c9d0eb0e8f42d139db230e826415bbcabd9f0e9faaaee909894", size = 144119, upload-time = "2026-05-10T18:16:11.771Z" }, - { url = "https://files.pythonhosted.org/packages/61/fd/caa1d60b12f7dd79ccea23054e06eeaebe266a5f52c40a6b651069200ce5/librt-0.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fa475675db22290c3158e1d42326d0f5a65f04f44a0e68c3630a25b53560fb9c", size = 143565, upload-time = "2026-05-10T18:16:13.334Z" }, - { url = "https://files.pythonhosted.org/packages/b8/a9/dc744f5c2b4978d48db970be29f22716d3413d28b14ad99740817315cf2c/librt-0.11.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:621db29691044bdeda22e789e482e1b0f3a985d90e3426c9c6d17606416205ea", size = 485395, upload-time = "2026-05-10T18:16:14.729Z" }, - { url = "https://files.pythonhosted.org/packages/8f/21/7f8e97a1e4dae952a5a95948f6f8507a173bc1e669f54340bba6ca1ca31b/librt-0.11.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:a9010e2ed5b3a9e158c5fd966b3ab7e834bb3d3aacc8f66c91dd4b57a3799230", size = 479383, upload-time = "2026-05-10T18:16:16.321Z" }, - { url = "https://files.pythonhosted.org/packages/a6/6d/d8ee9c114bebf2c50e29ec2aa940826fccb62a645c3e4c18760987d0e16d/librt-0.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c39513d8b7477a2e1ed8c43fc21c524e8d5a0f8d4e8b7b074dbdbe7820a08e2", size = 513010, upload-time = "2026-05-10T18:16:17.647Z" }, - { url = "https://files.pythonhosted.org/packages/f0/43/0b5708af2bd30a46400e72ba6bdaa8f066f15fb9a688527e34220e8d6c06/librt-0.11.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7aef3cf1d5af86e770ab04bfd993dfc4ae8b8c17f66fb77dd4a7d50de7bbb1a3", size = 508433, upload-time = "2026-05-10T18:16:19.309Z" }, - { url = "https://files.pythonhosted.org/packages/4a/50/356187247d09013490481033183b3532b58acf8028bcb34b2b56a375c9b2/librt-0.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:557183ddc36babe46b27dd60facbd5adb4492181a5be887587d57cda6e092f21", size = 522595, upload-time = "2026-05-10T18:16:20.642Z" }, - { url = "https://files.pythonhosted.org/packages/40/e7/c6ac4240899c7f3248079d5a9900debe0dadb3fdeaf856684c987105ba47/librt-0.11.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:83d3e1f72bd42f6c5c0b7daec530c3f829bd02db42c70b8ddf0c2d90a2459930", size = 527255, upload-time = "2026-05-10T18:16:22.352Z" }, - { url = "https://files.pythonhosted.org/packages/eb/b5/a81322dbeedeeaf9c1ee6f001734d28a09d8383ac9e6779bc24bbd0743c6/librt-0.11.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:4ce1f21fbe589bc1afd7872dece84fb0e1144f794a288e58a10d2c54a55c43be", size = 516847, upload-time = "2026-05-10T18:16:23.627Z" }, - { url = "https://files.pythonhosted.org/packages/ae/66/6e6323787d592b55204a42595ff1102da5115601b53a7e9ddebc889a6da5/librt-0.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b09f7044ea2b64c9da42fd3d335666518cfd1c6e8a182c95da73d0214b41e", size = 553920, upload-time = "2026-05-10T18:16:25.025Z" }, - { url = "https://files.pythonhosted.org/packages/9c/21/623f8ca230857102066d9ca8c6c1734995908c4d0d1bee7bb2ef0021cb33/librt-0.11.0-cp313-cp313-win32.whl", hash = "sha256:78fddc31cd4d3caa897ad5d31f856b1faadc9474021ad6cb182b9018793e254e", size = 101898, upload-time = "2026-05-10T18:16:26.649Z" }, - { url = "https://files.pythonhosted.org/packages/b3/1d/b4ebd44dd723f768469007515cb92251e0ae286c94c140f374801140fa74/librt-0.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:8ca8aa88751a775870b764e93bad5135385f563cb8dcee399abf034ea4d3cb47", size = 119812, upload-time = "2026-05-10T18:16:27.859Z" }, - { url = "https://files.pythonhosted.org/packages/3b/e4/b2f4ca7965ca373b491cdb4bc25cdb30c1649ca81a8782056a83850292a9/librt-0.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:96f044bb325fd9cf1a723015638c219e9143f0dfbc0ca54c565df2b7fc748b44", size = 103448, upload-time = "2026-05-10T18:16:29.066Z" }, - { url = "https://files.pythonhosted.org/packages/29/eb/dbce197da4e227779e56b5735f2decc3eb36e55a1cdbf1bd65d6639d76c1/librt-0.11.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4a017a95e5837dc15a8c5661d60e05daa96b90908b1aa6b7acdf443cd25c8ebd", size = 143345, upload-time = "2026-05-10T18:16:30.674Z" }, - { url = "https://files.pythonhosted.org/packages/76/a3/254bebd0c11c8ba684018efb8006ff22e466abce445215cca6c778e7d9de/librt-0.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b1ecbd9819deccc39b7542bf4d2a740d8a620694d39989e58661d3763458f8d4", size = 143131, upload-time = "2026-05-10T18:16:32.037Z" }, - { url = "https://files.pythonhosted.org/packages/f1/3f/f77d6122d21ac7bf6ae8a7dfced1bd2a7ac545d3273ebdcaf8042f6d619f/librt-0.11.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7da327dacd7be8f8ec36547373550744a3cc0e536d54665cd83f8bcd961200e8", size = 477024, upload-time = "2026-05-10T18:16:33.493Z" }, - { url = "https://files.pythonhosted.org/packages/ac/0a/2c996dadebaa7d9bbbd43ef2d4f3e66b6da545f838a41694ef6172cebec8/librt-0.11.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:0dc56b1f8d06e60db362cc3fdae206681817f86ce4725d34511473487f12a34b", size = 474221, upload-time = "2026-05-10T18:16:34.864Z" }, - { url = "https://files.pythonhosted.org/packages/0a/7e/f5d92af8486b8272c23b3e686b46ff72d89c8169585eb61eef01a2ac7147/librt-0.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05fb8fb2ab90e21c8d12ea240d744ad514da9baf381ebfa70d91d20d21713175", size = 505174, upload-time = "2026-05-10T18:16:36.705Z" }, - { url = "https://files.pythonhosted.org/packages/af/1a/cb0734fe86398eb33193ab753b7326255c74cac5eb09e76b9b16536e7adb/librt-0.11.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae74872be221df4374d10fec61f93ed1513b9546ea84f2c0bf73ab3e9bd0b03", size = 497216, upload-time = "2026-05-10T18:16:38.418Z" }, - { url = "https://files.pythonhosted.org/packages/18/06/094820f91558b66e29943c0ec41c9914f460f48dd51fc503c3101e10842d/librt-0.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:32bcc918c0148eb7e3d57385125bac7e5f9e4359d05f07448b09f6f778c2f31c", size = 513921, upload-time = "2026-05-10T18:16:39.848Z" }, - { url = "https://files.pythonhosted.org/packages/0b/c2/00de9018871a282f530cacb457d5ec0428f6ac7e6fedde9aff7468d9fb04/librt-0.11.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f9743fc99135d5f78d2454435615f6dec0473ca507c26ce9d92b10b562a280d3", size = 520850, upload-time = "2026-05-10T18:16:41.471Z" }, - { url = "https://files.pythonhosted.org/packages/51/9d/64631832348fd1834fb3a61b996434edddaaf25a31d03b0a76273159d2cf/librt-0.11.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5ba067f4aadae8fda802d91d2124c90c42195ff32d9161d3549e6d05cfe26f96", size = 504237, upload-time = "2026-05-10T18:16:43.15Z" }, - { url = "https://files.pythonhosted.org/packages/a5/ec/ae5525eb16edc827a044e7bb8777a455ff95d4bca9379e7e6bddd7383647/librt-0.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:de3bf945454d032f9e390b85c4072e0a0570bf825421c8be0e71209fa65e1abe", size = 546261, upload-time = "2026-05-10T18:16:44.408Z" }, - { url = "https://files.pythonhosted.org/packages/5a/09/adce371f27ca039411da9659f7430fcc2ba6cd0c7b3e4467a0f091be7fa9/librt-0.11.0-cp314-cp314-win32.whl", hash = "sha256:d2277a05f6dcb9fd13db9566aac4fabd68c3ea1ea46ee5567d4eef8efa495a2f", size = 96965, upload-time = "2026-05-10T18:16:46.039Z" }, - { url = "https://files.pythonhosted.org/packages/d6/ee/8ac720d98548f173c7ce2e632a7ca94673f74cacd5c8162a84af5b35958a/librt-0.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:ab73e8db5e3f564d812c1f5c3a175930a5f9bc96ccb5e3b22a34d7858b401cf7", size = 115151, upload-time = "2026-05-10T18:16:47.133Z" }, - { url = "https://files.pythonhosted.org/packages/94/20/c900cf14efeb09b6bef2b2dff20779f73464b97fd58d1c6bccc379588ae3/librt-0.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:aea3caa317752e3a466fa8af45d91ee0ea8c7fdd96e42b0a8dd9b76a7931eba1", size = 98850, upload-time = "2026-05-10T18:16:48.597Z" }, - { url = "https://files.pythonhosted.org/packages/0c/71/944bfe4b64e12abffcd3c15e1cce07f72f3d55655083786285f4dedeb532/librt-0.11.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:d1b36540d7aaf9b9101b3a6f376c8d8e9f7a9aec93ed05918f2c69d493ffef72", size = 151138, upload-time = "2026-05-10T18:16:49.839Z" }, - { url = "https://files.pythonhosted.org/packages/b6/10/99e64a5c86989357fda078c8143c533389585f6473b7439172dd8f3b3b2d/librt-0.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:efbb343ab2ce3540f4ecbe6315d677ed70f37cd9a72b1e58066c918ca83acbaa", size = 151976, upload-time = "2026-05-10T18:16:51.062Z" }, - { url = "https://files.pythonhosted.org/packages/21/31/5072ad880946d83e5ea4147d6d018c78eefce85b77819b19bdd0ee229435/librt-0.11.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa0dd688aab3f7914d3e6e5e3554978e0383312fb8e771d84be008a35b9ee548", size = 557927, upload-time = "2026-05-10T18:16:52.632Z" }, - { url = "https://files.pythonhosted.org/packages/5e/8d/70b5fb7cfbab60edbe7381614ab985da58e144fbf465c86d44c95f43cdca/librt-0.11.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:f5fb36b8c6c63fdcbb1d526d94c0d1331610d43f4118cc1beb4efef4f3faacb2", size = 539698, upload-time = "2026-05-10T18:16:53.934Z" }, - { url = "https://files.pythonhosted.org/packages/fa/a3/ba3495a0b3edbd24a4cae0d1d3c64f39a9fc45d06e812101289b50c1a619/librt-0.11.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4a9a237d13addb93715b6fee74023d5ee3469b53fce527626c0e088aa585805f", size = 577162, upload-time = "2026-05-10T18:16:55.589Z" }, - { url = "https://files.pythonhosted.org/packages/f7/db/36e25fb81f99937ff1b96612a1dc9fd66f039cb9cc3aee12c01fac31aab9/librt-0.11.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5ddd17bd87b2c56ddd60e546a7984a2e64c4e8eab92fb4cf3830a48ad5469d51", size = 566494, upload-time = "2026-05-10T18:16:56.975Z" }, - { url = "https://files.pythonhosted.org/packages/33/0d/3f622b47f0b013eeb9cf4cc07ae9bfe378d832a4eec998b2b209fe84244d/librt-0.11.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd43992b4473d42f12ff9e68326079f0696d9d4e6000e8f39a0238d482ba6ee2", size = 596858, upload-time = "2026-05-10T18:16:58.374Z" }, - { url = "https://files.pythonhosted.org/packages/a9/02/71b90bc93039c46a2000651f6ad60122b114c8f54c4ad306e0e96f5b75ad/librt-0.11.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:f8e3e8056dd674e279741485e2e512d6e9a751c7455809d0114e6ebf8d781085", size = 590318, upload-time = "2026-05-10T18:16:59.676Z" }, - { url = "https://files.pythonhosted.org/packages/04/04/418cb3f75621e2b761fb1ab0f017f4d70a1a72a6e7c74ee4f7e8d198c2f3/librt-0.11.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c1f708d8ae9c56cf38a903c44297243d2ec83fd82b396b977e0144a3e76217e3", size = 575115, upload-time = "2026-05-10T18:17:01.007Z" }, - { url = "https://files.pythonhosted.org/packages/cc/2c/5a2183ac58dd911f26b5d7e7d7d8f1d87fcecdddd99d6c12169a258ff62c/librt-0.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0add982e0e7b9fc14cf4b33789d5f13f66581889b88c2f58099f6ce8f92617bd", size = 617918, upload-time = "2026-05-10T18:17:02.682Z" }, - { url = "https://files.pythonhosted.org/packages/15/1f/dc6771a52592a4451be6effa200cbfc9cec61e4393d3033d81a9d307961d/librt-0.11.0-cp314-cp314t-win32.whl", hash = "sha256:2b481d846ac894c4e8403c5fd0e87c5d11d6499e404b474602508a224ff531c8", size = 103562, upload-time = "2026-05-10T18:17:03.99Z" }, - { url = "https://files.pythonhosted.org/packages/62/4a/7d1415567027286a75ba1093ec4aca11f073e0f559c530cf3e0a757ad55c/librt-0.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:28edb433edde181112a908c78907af28f964eabc15f4dd16c9d66c834302677c", size = 124327, upload-time = "2026-05-10T18:17:05.465Z" }, - { url = "https://files.pythonhosted.org/packages/ce/62/b40b382fa0c66fee1478073eb8db352a4a6beda4a1adccf1df911d8c289c/librt-0.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dee008f20b542e3cd162ba338a7f9ec0f6d23d395f66fe8aeeec3c9d067ea253", size = 102572, upload-time = "2026-05-10T18:17:06.809Z" }, + { url = "https://files.pythonhosted.org/packages/f3/4a/c64265d71b84030174ff3ac2cd16d8b664072afab8c41fccd8e2ee5a6f8d/librt-0.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f8e12706dcb8ff6b3ed57514a19e45c49ad00bcd423e87b2b2e4b5f64578443", size = 67529, upload-time = "2026-04-09T16:04:27.373Z" }, + { url = "https://files.pythonhosted.org/packages/23/b1/30ca0b3a8bdac209a00145c66cf42e5e7da2cc056ffc6ebc5c7b430ddd34/librt-0.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4e3dda8345307fd7306db0ed0cb109a63a2c85ba780eb9dc2d09b2049a931f9c", size = 70248, upload-time = "2026-04-09T16:04:28.758Z" }, + { url = "https://files.pythonhosted.org/packages/fa/fc/c6018dc181478d6ac5aa24a5846b8185101eb90894346db239eb3ea53209/librt-0.9.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:de7dac64e3eb832ffc7b840eb8f52f76420cde1b845be51b2a0f6b870890645e", size = 202184, upload-time = "2026-04-09T16:04:29.893Z" }, + { url = "https://files.pythonhosted.org/packages/bf/58/d69629f002203370ef41ea69ff71c49a2c618aec39b226ff49986ecd8623/librt-0.9.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22a904cbdb678f7cb348c90d543d3c52f581663d687992fee47fd566dcbf5285", size = 212926, upload-time = "2026-04-09T16:04:31.126Z" }, + { url = "https://files.pythonhosted.org/packages/cc/55/01d859f57824e42bd02465c77bec31fa5ef9d8c2bcee702ccf8ef1b9f508/librt-0.9.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:224b9727eb8bc188bc3bcf29d969dba0cd61b01d9bac80c41575520cc4baabb2", size = 225664, upload-time = "2026-04-09T16:04:32.352Z" }, + { url = "https://files.pythonhosted.org/packages/9b/02/32f63ad0ef085a94a70315291efe1151a48b9947af12261882f8445b2a30/librt-0.9.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e94cbc6ad9a6aeea46d775cbb11f361022f778a9cc8cc90af653d3a594b057ce", size = 219534, upload-time = "2026-04-09T16:04:33.667Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5a/9d77111a183c885acf3b3b6e4c00f5b5b07b5817028226499a55f1fedc59/librt-0.9.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7bc30ad339f4e1a01d4917d645e522a0bc0030644d8973f6346397c93ba1503f", size = 227322, upload-time = "2026-04-09T16:04:34.945Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e7/05d700c93063753e12ab230b972002a3f8f3b9c95d8a980c2f646c8b6963/librt-0.9.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:56d65b583cf43b8cf4c8fbe1e1da20fa3076cc32a1149a141507af1062718236", size = 223407, upload-time = "2026-04-09T16:04:36.22Z" }, + { url = "https://files.pythonhosted.org/packages/c0/26/26c3124823c67c987456977c683da9a27cc874befc194ddcead5f9988425/librt-0.9.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0a1be03168b2691ba61927e299b352a6315189199ca18a57b733f86cb3cc8d38", size = 221302, upload-time = "2026-04-09T16:04:37.62Z" }, + { url = "https://files.pythonhosted.org/packages/50/2b/c7cc2be5cf4ff7b017d948a789256288cb33a517687ff1995e72a7eea79f/librt-0.9.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:63c12efcd160e1d14da11af0c46c0217473e1e0d2ae1acbccc83f561ea4c2a7b", size = 243893, upload-time = "2026-04-09T16:04:38.909Z" }, + { url = "https://files.pythonhosted.org/packages/62/d3/da553d37417a337d12660450535d5fd51373caffbedf6962173c87867246/librt-0.9.0-cp310-cp310-win32.whl", hash = "sha256:e9002e98dcb1c0a66723592520decd86238ddcef168b37ff6cfb559200b4b774", size = 55375, upload-time = "2026-04-09T16:04:40.148Z" }, + { url = "https://files.pythonhosted.org/packages/9b/5a/46fa357bab8311b6442a83471591f2f9e5b15ecc1d2121a43725e0c529b8/librt-0.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:9fcb461fbf70654a52a7cc670e606f04449e2374c199b1825f754e16dacfedd8", size = 62581, upload-time = "2026-04-09T16:04:41.452Z" }, + { url = "https://files.pythonhosted.org/packages/e2/1e/2ec7afcebcf3efea593d13aee18bbcfdd3a243043d848ebf385055e9f636/librt-0.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:90904fac73c478f4b83f4ed96c99c8208b75e6f9a8a1910548f69a00f1eaa671", size = 67155, upload-time = "2026-04-09T16:04:42.933Z" }, + { url = "https://files.pythonhosted.org/packages/18/77/72b85afd4435268338ad4ec6231b3da8c77363f212a0227c1ff3b45e4d35/librt-0.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:789fff71757facc0738e8d89e3b84e4f0251c1c975e85e81b152cdaca927cc2d", size = 69916, upload-time = "2026-04-09T16:04:44.042Z" }, + { url = "https://files.pythonhosted.org/packages/27/fb/948ea0204fbe2e78add6d46b48330e58d39897e425560674aee302dca81c/librt-0.9.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1bf465d1e5b0a27713862441f6467b5ab76385f4ecf8f1f3a44f8aa3c695b4b6", size = 199635, upload-time = "2026-04-09T16:04:45.5Z" }, + { url = "https://files.pythonhosted.org/packages/ac/cd/894a29e251b296a27957856804cfd21e93c194aa131de8bb8032021be07e/librt-0.9.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f819e0c6413e259a17a7c0d49f97f405abadd3c2a316a3b46c6440b7dbbedbb1", size = 211051, upload-time = "2026-04-09T16:04:47.016Z" }, + { url = "https://files.pythonhosted.org/packages/18/8f/dcaed0bc084a35f3721ff2d081158db569d2c57ea07d35623ddaca5cfc8e/librt-0.9.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0785c2fb4a81e1aece366aa3e2e039f4a4d7d21aaaded5227d7f3c703427882", size = 224031, upload-time = "2026-04-09T16:04:48.207Z" }, + { url = "https://files.pythonhosted.org/packages/03/44/88f6c1ed1132cd418601cc041fbd92fed28b3a09f39de81978e0822d13ff/librt-0.9.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:80b25c7b570a86c03b5da69e665809deb39265476e8e21d96a9328f9762f9990", size = 218069, upload-time = "2026-04-09T16:04:50.025Z" }, + { url = "https://files.pythonhosted.org/packages/a3/90/7d02e981c2db12188d82b4410ff3e35bfdb844b26aecd02233626f46af2b/librt-0.9.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d4d16b608a1c43d7e33142099a75cd93af482dadce0bf82421e91cad077157f4", size = 224857, upload-time = "2026-04-09T16:04:51.684Z" }, + { url = "https://files.pythonhosted.org/packages/ef/c3/c77e706b7215ca32e928d47535cf13dbc3d25f096f84ddf8fbc06693e229/librt-0.9.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:194fc1a32e1e21fe809d38b5faea66cc65eaa00217c8901fbdb99866938adbdb", size = 219865, upload-time = "2026-04-09T16:04:52.949Z" }, + { url = "https://files.pythonhosted.org/packages/52/d1/32b0c1a0eb8461c70c11656c46a29f760b7c7edf3c36d6f102470c17170f/librt-0.9.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8c6bc1384d9738781cfd41d09ad7f6e8af13cfea2c75ece6bd6d2566cdea2076", size = 218451, upload-time = "2026-04-09T16:04:54.174Z" }, + { url = "https://files.pythonhosted.org/packages/74/d1/adfd0f9c44761b1d49b1bec66173389834c33ee2bd3c7fd2e2367f1942d4/librt-0.9.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:15cb151e52a044f06e54ac7f7b47adbfc89b5c8e2b63e1175a9d587c43e8942a", size = 241300, upload-time = "2026-04-09T16:04:55.452Z" }, + { url = "https://files.pythonhosted.org/packages/09/b0/9074b64407712f0003c27f5b1d7655d1438979155f049720e8a1abd9b1a1/librt-0.9.0-cp311-cp311-win32.whl", hash = "sha256:f100bfe2acf8a3689af9d0cc660d89f17286c9c795f9f18f7b62dd1a6b247ae6", size = 55668, upload-time = "2026-04-09T16:04:56.689Z" }, + { url = "https://files.pythonhosted.org/packages/24/19/40b77b77ce80b9389fb03971431b09b6b913911c38d412059e0b3e2a9ef2/librt-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:0b73e4266307e51c95e09c0750b7ec383c561d2e97d58e473f6f6a209952fbb8", size = 62976, upload-time = "2026-04-09T16:04:57.733Z" }, + { url = "https://files.pythonhosted.org/packages/70/9d/9fa7a64041e29035cb8c575af5f0e3840be1b97b4c4d9061e0713f171849/librt-0.9.0-cp311-cp311-win_arm64.whl", hash = "sha256:bc5518873822d2faa8ebdd2c1a4d7c8ef47b01a058495ab7924cb65bdbf5fc9a", size = 53502, upload-time = "2026-04-09T16:04:58.806Z" }, + { url = "https://files.pythonhosted.org/packages/bf/90/89ddba8e1c20b0922783cd93ed8e64f34dc05ab59c38a9c7e313632e20ff/librt-0.9.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9b3e3bc363f71bda1639a4ee593cb78f7fbfeacc73411ec0d4c92f00730010a4", size = 68332, upload-time = "2026-04-09T16:05:00.09Z" }, + { url = "https://files.pythonhosted.org/packages/a8/40/7aa4da1fb08bdeeb540cb07bfc8207cb32c5c41642f2594dbd0098a0662d/librt-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0a09c2f5869649101738653a9b7ab70cf045a1105ac66cbb8f4055e61df78f2d", size = 70581, upload-time = "2026-04-09T16:05:01.213Z" }, + { url = "https://files.pythonhosted.org/packages/48/ac/73a2187e1031041e93b7e3a25aae37aa6f13b838c550f7e0f06f66766212/librt-0.9.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5ca8e133d799c948db2ab1afc081c333a825b5540475164726dcbf73537e5c2f", size = 203984, upload-time = "2026-04-09T16:05:02.542Z" }, + { url = "https://files.pythonhosted.org/packages/5e/3d/23460d571e9cbddb405b017681df04c142fb1b04cbfce77c54b08e28b108/librt-0.9.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:603138ee838ee1583f1b960b62d5d0007845c5c423feb68e44648b1359014e27", size = 215762, upload-time = "2026-04-09T16:05:04.127Z" }, + { url = "https://files.pythonhosted.org/packages/de/1e/42dc7f8ab63e65b20640d058e63e97fd3e482c1edbda3570d813b4d0b927/librt-0.9.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4003f70c56a5addd6aa0897f200dd59afd3bf7bcd5b3cce46dd21f925743bc2", size = 230288, upload-time = "2026-04-09T16:05:05.883Z" }, + { url = "https://files.pythonhosted.org/packages/dc/08/ca812b6d8259ad9ece703397f8ad5c03af5b5fedfce64279693d3ce4087c/librt-0.9.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:78042f6facfd98ecb25e9829c7e37cce23363d9d7c83bc5f72702c5059eb082b", size = 224103, upload-time = "2026-04-09T16:05:07.148Z" }, + { url = "https://files.pythonhosted.org/packages/b6/3f/620490fb2fa66ffd44e7f900254bc110ebec8dac6c1b7514d64662570e6f/librt-0.9.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a361c9434a64d70a7dbb771d1de302c0cc9f13c0bffe1cf7e642152814b35265", size = 232122, upload-time = "2026-04-09T16:05:08.386Z" }, + { url = "https://files.pythonhosted.org/packages/e9/83/12864700a1b6a8be458cf5d05db209b0d8e94ae281e7ec261dbe616597b4/librt-0.9.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:dd2c7e082b0b92e1baa4da28163a808672485617bc855cc22a2fd06978fa9084", size = 225045, upload-time = "2026-04-09T16:05:09.707Z" }, + { url = "https://files.pythonhosted.org/packages/fd/1b/845d339c29dc7dbc87a2e992a1ba8d28d25d0e0372f9a0a2ecebde298186/librt-0.9.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7e6274fd33fc5b2a14d41c9119629d3ff395849d8bcbc80cf637d9e8d2034da8", size = 227372, upload-time = "2026-04-09T16:05:10.942Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fe/277985610269d926a64c606f761d58d3db67b956dbbf40024921e95e7fcb/librt-0.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5093043afb226ecfa1400120d1ebd4442b4f99977783e4f4f7248879009b227f", size = 248224, upload-time = "2026-04-09T16:05:12.254Z" }, + { url = "https://files.pythonhosted.org/packages/92/1b/ee486d244b8de6b8b5dbaefabe6bfdd4a72e08f6353edf7d16d27114da8d/librt-0.9.0-cp312-cp312-win32.whl", hash = "sha256:9edcc35d1cae9fd5320171b1a838c7da8a5c968af31e82ecc3dff30b4be0957f", size = 55986, upload-time = "2026-04-09T16:05:13.529Z" }, + { url = "https://files.pythonhosted.org/packages/89/7a/ba1737012308c17dc6d5516143b5dce9a2c7ba3474afd54e11f44a4d1ef3/librt-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc2917258e131ae5f958a4d872e07555b51cb7466a43433218061c74ef33745", size = 63260, upload-time = "2026-04-09T16:05:14.68Z" }, + { url = "https://files.pythonhosted.org/packages/36/e4/01752c113da15127f18f7bf11142f5640038f062407a611c059d0036c6aa/librt-0.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:90e6d5420fc8a300518d4d2288154ff45005e920425c22cbbfe8330f3f754bd9", size = 53694, upload-time = "2026-04-09T16:05:16.095Z" }, + { url = "https://files.pythonhosted.org/packages/5f/d7/1b3e26fffde1452d82f5666164858a81c26ebe808e7ae8c9c88628981540/librt-0.9.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f29b68cd9714531672db62cc54f6e8ff981900f824d13fa0e00749189e13778e", size = 68367, upload-time = "2026-04-09T16:05:17.243Z" }, + { url = "https://files.pythonhosted.org/packages/a5/5b/c61b043ad2e091fbe1f2d35d14795e545d0b56b03edaa390fa1dcee3d160/librt-0.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7d5c8a5929ac325729f6119802070b561f4db793dffc45e9ac750992a4ed4d22", size = 70595, upload-time = "2026-04-09T16:05:18.471Z" }, + { url = "https://files.pythonhosted.org/packages/a3/22/2448471196d8a73370aa2f23445455dc42712c21404081fcd7a03b9e0749/librt-0.9.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:756775d25ec8345b837ab52effee3ad2f3b2dfd6bbee3e3f029c517bd5d8f05a", size = 204354, upload-time = "2026-04-09T16:05:19.593Z" }, + { url = "https://files.pythonhosted.org/packages/ac/5e/39fc4b153c78cfd2c8a2dcb32700f2d41d2312aa1050513183be4540930d/librt-0.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b8f5d00b49818f4e2b1667db994488b045835e0ac16fe2f924f3871bd2b8ac5", size = 216238, upload-time = "2026-04-09T16:05:20.868Z" }, + { url = "https://files.pythonhosted.org/packages/d7/42/bc2d02d0fa7badfa63aa8d6dcd8793a9f7ef5a94396801684a51ed8d8287/librt-0.9.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c81aef782380f0f13ead670aae01825eb653b44b046aa0e5ebbb79f76ed4aa11", size = 230589, upload-time = "2026-04-09T16:05:22.305Z" }, + { url = "https://files.pythonhosted.org/packages/c8/7b/e2d95cc513866373692aa5edf98080d5602dd07cabfb9e5d2f70df2f25f7/librt-0.9.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66b58fed90a545328e80d575467244de3741e088c1af928f0b489ebec3ef3858", size = 224610, upload-time = "2026-04-09T16:05:23.647Z" }, + { url = "https://files.pythonhosted.org/packages/31/d5/6cec4607e998eaba57564d06a1295c21b0a0c8de76e4e74d699e627bd98c/librt-0.9.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e78fb7419e07d98c2af4b8567b72b3eaf8cb05caad642e9963465569c8b2d87e", size = 232558, upload-time = "2026-04-09T16:05:25.025Z" }, + { url = "https://files.pythonhosted.org/packages/95/8c/27f1d8d3aaf079d3eb26439bf0b32f1482340c3552e324f7db9dca858671/librt-0.9.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2c3786f0f4490a5cd87f1ed6cefae833ad6b1060d52044ce0434a2e85893afd0", size = 225521, upload-time = "2026-04-09T16:05:26.311Z" }, + { url = "https://files.pythonhosted.org/packages/6b/d8/1e0d43b1c329b416017619469b3c3801a25a6a4ef4a1c68332aeaa6f72ca/librt-0.9.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8494cfc61e03542f2d381e71804990b3931175a29b9278fdb4a5459948778dc2", size = 227789, upload-time = "2026-04-09T16:05:27.624Z" }, + { url = "https://files.pythonhosted.org/packages/2c/b4/d3d842e88610fcd4c8eec7067b0c23ef2d7d3bff31496eded6a83b0f99be/librt-0.9.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:07cf11f769831186eeac424376e6189f20ace4f7263e2134bdb9757340d84d4d", size = 248616, upload-time = "2026-04-09T16:05:29.181Z" }, + { url = "https://files.pythonhosted.org/packages/ec/28/527df8ad0d1eb6c8bdfa82fc190f1f7c4cca5a1b6d7b36aeabf95b52d74d/librt-0.9.0-cp313-cp313-win32.whl", hash = "sha256:850d6d03177e52700af605fd60db7f37dcb89782049a149674d1a9649c2138fd", size = 56039, upload-time = "2026-04-09T16:05:30.709Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a7/413652ad0d92273ee5e30c000fc494b361171177c83e57c060ecd3c21538/librt-0.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:a5af136bfba820d592f86c67affcef9b3ff4d4360ac3255e341e964489b48519", size = 63264, upload-time = "2026-04-09T16:05:31.881Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0a/92c244309b774e290ddb15e93363846ae7aa753d9586b8aad511c5e6145b/librt-0.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:4c4d0440a3a8e31d962340c3e1cc3fc9ee7febd34c8d8f770d06adb947779ea5", size = 53728, upload-time = "2026-04-09T16:05:33.31Z" }, + { url = "https://files.pythonhosted.org/packages/cd/c1/184e539543f06ea2912f4b92a5ffaede4f9b392689e3f00acbf8134bee92/librt-0.9.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:3f05d145df35dca5056a8bc3838e940efebd893a54b3e19b2dda39ceaa299bcb", size = 67830, upload-time = "2026-04-09T16:05:34.517Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ad/23399bdcb7afca819acacdef31b37ee59de261bd66b503a7995c03c4b0dc/librt-0.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1c587494461ebd42229d0f1739f3aa34237dd9980623ecf1be8d3bcba79f4499", size = 70280, upload-time = "2026-04-09T16:05:35.649Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0b/4542dc5a2b8772dbf92cafb9194701230157e73c14b017b6961a23598b03/librt-0.9.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b0a2040f801406b93657a70b72fa12311063a319fee72ce98e1524da7200171f", size = 201925, upload-time = "2026-04-09T16:05:36.739Z" }, + { url = "https://files.pythonhosted.org/packages/31/d4/8ee7358b08fd0cfce051ef96695380f09b3c2c11b77c9bfbc367c921cce5/librt-0.9.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f38bc489037eca88d6ebefc9c4d41a4e07c8e8b4de5188a9e6d290273ad7ebb1", size = 212381, upload-time = "2026-04-09T16:05:38.043Z" }, + { url = "https://files.pythonhosted.org/packages/f2/94/a2025fe442abedf8b038038dab3dba942009ad42b38ea064a1a9e6094241/librt-0.9.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3fd278f5e6bf7c75ccd6d12344eb686cc020712683363b66f46ac79d37c799f", size = 227065, upload-time = "2026-04-09T16:05:39.394Z" }, + { url = "https://files.pythonhosted.org/packages/7c/e9/b9fcf6afa909f957cfbbf918802f9dada1bd5d3c1da43d722fd6a310dc3f/librt-0.9.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fcbdf2a9ca24e87bbebb47f1fe34e531ef06f104f98c9ccfc953a3f3344c567a", size = 221333, upload-time = "2026-04-09T16:05:40.999Z" }, + { url = "https://files.pythonhosted.org/packages/ac/7c/ba54cd6aa6a3c8cd12757a6870e0c79a64b1e6327f5248dcff98423f4d43/librt-0.9.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e306d956cfa027fe041585f02a1602c32bfa6bb8ebea4899d373383295a6c62f", size = 229051, upload-time = "2026-04-09T16:05:42.605Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4b/8cfdbad314c8677a0148bf0b70591d6d18587f9884d930276098a235461b/librt-0.9.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:465814ab157986acb9dfa5ccd7df944be5eefc0d08d31ec6e8d88bc71251d845", size = 222492, upload-time = "2026-04-09T16:05:43.842Z" }, + { url = "https://files.pythonhosted.org/packages/1f/d1/2eda69563a1a88706808decdce035e4b32755dbfbb0d05e1a65db9547ed1/librt-0.9.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:703f4ae36d6240bfe24f542bac784c7e4194ec49c3ba5a994d02891649e2d85b", size = 223849, upload-time = "2026-04-09T16:05:45.054Z" }, + { url = "https://files.pythonhosted.org/packages/04/44/b2ed37df6be5b3d42cfe36318e0598e80843d5c6308dd63d0bf4e0ce5028/librt-0.9.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3be322a15ee5e70b93b7a59cfd074614f22cc8c9ff18bd27f474e79137ea8d3b", size = 245001, upload-time = "2026-04-09T16:05:46.34Z" }, + { url = "https://files.pythonhosted.org/packages/47/e7/617e412426df89169dd2a9ed0cc8752d5763336252c65dbf945199915119/librt-0.9.0-cp314-cp314-win32.whl", hash = "sha256:b8da9f8035bb417770b1e1610526d87ad4fc58a2804dc4d79c53f6d2cf5a6eb9", size = 51799, upload-time = "2026-04-09T16:05:47.738Z" }, + { url = "https://files.pythonhosted.org/packages/24/ed/c22ca4db0ca3cbc285e4d9206108746beda561a9792289c3c31281d7e9df/librt-0.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:b8bd70d5d816566a580d193326912f4a76ec2d28a97dc4cd4cc831c0af8e330e", size = 59165, upload-time = "2026-04-09T16:05:49.198Z" }, + { url = "https://files.pythonhosted.org/packages/24/56/875398fafa4cbc8f15b89366fc3287304ddd3314d861f182a4b87595ace0/librt-0.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:fc5758e2b7a56532dc33e3c544d78cbaa9ecf0a0f2a2da2df882c1d6b99a317f", size = 49292, upload-time = "2026-04-09T16:05:50.362Z" }, + { url = "https://files.pythonhosted.org/packages/4c/61/bc448ecbf9b2d69c5cff88fe41496b19ab2a1cbda0065e47d4d0d51c0867/librt-0.9.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:f24b90b0e0c8cc9491fb1693ae91fe17cb7963153a1946395acdbdd5818429a4", size = 70175, upload-time = "2026-04-09T16:05:51.564Z" }, + { url = "https://files.pythonhosted.org/packages/60/f2/c47bb71069a73e2f04e70acbd196c1e5cc411578ac99039a224b98920fd4/librt-0.9.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3fe56e80badb66fdcde06bef81bbaa5bfcf6fbd7aefb86222d9e369c38c6b228", size = 72951, upload-time = "2026-04-09T16:05:52.699Z" }, + { url = "https://files.pythonhosted.org/packages/29/19/0549df59060631732df758e8886d92088da5fdbedb35b80e4643664e8412/librt-0.9.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:527b5b820b47a09e09829051452bb0d1dd2122261254e2a6f674d12f1d793d54", size = 225864, upload-time = "2026-04-09T16:05:53.895Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f8/3b144396d302ac08e50f89e64452c38db84bc7b23f6c60479c5d3abd303c/librt-0.9.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7d429bdd4ac0ab17c8e4a8af0ed2a7440b16eba474909ab357131018fe8c7e71", size = 241155, upload-time = "2026-04-09T16:05:55.191Z" }, + { url = "https://files.pythonhosted.org/packages/7a/ce/ee67ec14581de4043e61d05786d2aed6c9b5338816b7859bcf07455c6a9f/librt-0.9.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7202bdcac47d3a708271c4304a474a8605a4a9a4a709e954bf2d3241140aa938", size = 252235, upload-time = "2026-04-09T16:05:56.549Z" }, + { url = "https://files.pythonhosted.org/packages/8a/fa/0ead15daa2b293a54101550b08d4bafe387b7d4a9fc6d2b985602bae69b6/librt-0.9.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0d620e74897f8c2613b3c4e2e9c1e422eb46d2ddd07df540784d44117836af3", size = 244963, upload-time = "2026-04-09T16:05:57.858Z" }, + { url = "https://files.pythonhosted.org/packages/29/68/9fbf9a9aa704ba87689e40017e720aced8d9a4d2b46b82451d8142f91ec9/librt-0.9.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d69fc39e627908f4c03297d5a88d9284b73f4d90b424461e32e8c2485e21c283", size = 257364, upload-time = "2026-04-09T16:05:59.686Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8d/9d60869f1b6716c762e45f66ed945b1e5dd649f7377684c3b176ae424648/librt-0.9.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:c2640e23d2b7c98796f123ffd95cf2022c7777aa8a4a3b98b36c570d37e85eee", size = 247661, upload-time = "2026-04-09T16:06:00.938Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/a5c365093962310bfdb4f6af256f191085078ffb529b3f0cbebb5b33ebe2/librt-0.9.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:451daa98463b7695b0a30aa56bf637831ea559e7b8101ac2ef6382e8eb15e29c", size = 248238, upload-time = "2026-04-09T16:06:02.537Z" }, + { url = "https://files.pythonhosted.org/packages/a0/3c/2d34365177f412c9e19c0a29f969d70f5343f27634b76b765a54d8b27705/librt-0.9.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:928bd06eca2c2bbf4349e5b817f837509b0604342e65a502de1d50a7570afd15", size = 269457, upload-time = "2026-04-09T16:06:03.833Z" }, + { url = "https://files.pythonhosted.org/packages/bc/cd/de45b239ea3bdf626f982a00c14bfcf2e12d261c510ba7db62c5969a27cd/librt-0.9.0-cp314-cp314t-win32.whl", hash = "sha256:a9c63e04d003bc0fb6a03b348018b9a3002f98268200e22cc80f146beac5dc40", size = 52453, upload-time = "2026-04-09T16:06:05.229Z" }, + { url = "https://files.pythonhosted.org/packages/7f/f9/bfb32ae428aa75c0c533915622176f0a17d6da7b72b5a3c6363685914f70/librt-0.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f162af66a2ed3f7d1d161a82ca584efd15acd9c1cff190a373458c32f7d42118", size = 60044, upload-time = "2026-04-09T16:06:06.398Z" }, + { url = "https://files.pythonhosted.org/packages/aa/47/7d70414bcdbb3bc1f458a8d10558f00bbfdb24e5a11740fc8197e12c3255/librt-0.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:a4b25c6c25cac5d0d9d6d6da855195b254e0021e513e0249f0e3b444dc6e0e61", size = 50009, upload-time = "2026-04-09T16:06:07.995Z" }, ] [[package]] name = "markdown-it-py" -version = "4.2.0" +version = "4.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mdurl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, ] [[package]] @@ -619,61 +464,60 @@ wheels = [ [[package]] name = "mypy" -version = "2.1.0" +version = "1.20.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ast-serialize" }, { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, { name = "mypy-extensions" }, { name = "pathspec" }, { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/82/15/cca9d88503549ed6fedeaa1d448cdddd542ee8a490232d732e278036fbf2/mypy-2.1.0.tar.gz", hash = "sha256:81e76ad12c2d804512e9b13240d1588316531bfba07558286078bfbce9613633", size = 3898359, upload-time = "2026-05-11T18:37:36.237Z" } +sdist = { url = "https://files.pythonhosted.org/packages/04/af/e3d4b3e9ec91a0ff9aabfdb38692952acf49bbb899c2e4c29acb3a6da3ae/mypy-1.20.2.tar.gz", hash = "sha256:e8222c26daaafd9e8626dec58ae36029f82585890589576f769a650dd20fd665", size = 3817349, upload-time = "2026-04-21T17:12:28.473Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/71/d351dca3e9b30da2328ee9d445c88b8388072808ebfbc49eb69d30b67749/mypy-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:11a6beb180257a805961aea9ec591bbd0bd17f1e18d35b8456d57aee5bedfedc", size = 14778792, upload-time = "2026-05-11T18:36:23.605Z" }, - { url = "https://files.pythonhosted.org/packages/2f/45/7d51594b644c17c0bcf74ed8cd5fc33b324276d708e8506f220b70dab9d9/mypy-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8ef78c1d306bbf9a8a12f526c44902c9c28dffd6c52c52bf6a72641ce18d3849", size = 13645739, upload-time = "2026-05-11T18:37:22.752Z" }, - { url = "https://files.pythonhosted.org/packages/65/01/455c31b170e9468265074840bf18863a8482a24103fdaabe4e199392aa5f/mypy-2.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c209a90853081ff01d01ee895cafe10f7db1474e0d95beaeef0f6c1db9119bbd", size = 14074199, upload-time = "2026-05-11T18:35:09.292Z" }, - { url = "https://files.pythonhosted.org/packages/41/5a/93093f0b29a9e982deafde698f740a2eb2e05886e79ccf0594c7fd5413a3/mypy-2.1.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47cebf61abde7c088a4e27718a8b13a81655686b2e9c251f5c0915a802248166", size = 14953128, upload-time = "2026-05-11T18:31:57.678Z" }, - { url = "https://files.pythonhosted.org/packages/7f/2f/a196f5331d96170ad3d28f144d2aba690d4b2911381f68d51e489c7ab82a/mypy-2.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d57a90ae5e872138a425ec328edbc9b235d1934c4377881a33ec05b341acc9a8", size = 15249378, upload-time = "2026-05-11T18:33:00.101Z" }, - { url = "https://files.pythonhosted.org/packages/54/de/94d321cc12da9f71341ac0c270efbed5c725750c7b4c334d957de9a087d9/mypy-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:aea7f7a8a55b459c34275fc468ada6ca7c173a5e43a68f5dbe588a563d8a06b8", size = 11060994, upload-time = "2026-05-11T18:33:18.848Z" }, - { url = "https://files.pythonhosted.org/packages/e1/62/0c27ca55219a7c764a7fb88c7bb2b7b2f9780ade8bbf16bc8ed8400eef6b/mypy-2.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:c989640253f0d76843e9c6c1bbf4bd48c5e85ada61bde4beb37cb3eca035685e", size = 9976743, upload-time = "2026-05-11T18:31:25.554Z" }, - { url = "https://files.pythonhosted.org/packages/0a/a1/639f3024794a2a15899cb90707fe02e044c4412794c39c5769fd3df2e2ef/mypy-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a683016b16fe2f572dc04c72be7ee0504ac1605a265d0200f5cea695fb788f41", size = 14691685, upload-time = "2026-05-11T18:33:27.973Z" }, - { url = "https://files.pythonhosted.org/packages/3b/08/9a585dea4325f20d8b80dc78623fa50d1fd2173b710f6237afd6ba6ab39b/mypy-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1a293c534adb55271fef24a26da04b855540a8c13cc07bc5917b9fd2c394f2ca", size = 13555165, upload-time = "2026-05-11T18:32:16.107Z" }, - { url = "https://files.pythonhosted.org/packages/81/dc/7c42cc9c6cb01e8eb09961f1f738741d3e9c7e9d5c5b30ec69222625cd5f/mypy-2.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7406f4d048e71e576f5356d317e5b0a9e666dfd966bd99f9d14ca06e1a341538", size = 13994376, upload-time = "2026-05-11T18:32:39.256Z" }, - { url = "https://files.pythonhosted.org/packages/d4/fa/285946c33bce716e082c11dfeee9ee196eaf1f5042efb3581a31f9f205e4/mypy-2.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0210d626fc8b31ccc90233754c7bc90e1f43205e85d96387f7db1285b55c398", size = 14864618, upload-time = "2026-05-11T18:34:49.765Z" }, - { url = "https://files.pythonhosted.org/packages/2b/83/82397f48af6c27e295d57979ded8490c9829040152cf7571b2f026aeb9a0/mypy-2.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3712c20deed54e814eaaa825603bada8ea1c390670a397c95b98405347acc563", size = 15102063, upload-time = "2026-05-11T18:34:05.855Z" }, - { url = "https://files.pythonhosted.org/packages/40/68/b02dec39057b88eb03dc0aa854732e26e8361f34f9d0e20c7614967d1eba/mypy-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fcaa0e479066e31f7cceb6a3bea39cb22b2ff51a6b2f24f193d19179ba17c389", size = 11060564, upload-time = "2026-05-11T18:35:36.494Z" }, - { url = "https://files.pythonhosted.org/packages/cf/a8/ea3dcbef31f99b634f2ee23bb0321cbc8c1b388b76a861eb849f13c347dc/mypy-2.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:0b1a5260c95aa443083f9ed3592662941951bca3d4ca224a5dc517c38b7cf666", size = 9966983, upload-time = "2026-05-11T18:37:14.139Z" }, - { url = "https://files.pythonhosted.org/packages/95/b1/55861beb5c339b44f9a2ba92df9e2cb1eeb4ae1eee674cdf7772c797778b/mypy-2.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:244358bf1c0da7722230bce60683d52e8e9fd030554926f15b747a84efb5b3af", size = 14874381, upload-time = "2026-05-11T18:37:31.784Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b3/b7f770114b7d0ac92d0f76e8d93c2780844a70488a90e91821927850da86/mypy-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4ec7c57657493c7a75534df2751c8ae2cda383c16ecc55d2106c54476b1b16f6", size = 13665501, upload-time = "2026-05-11T18:34:23.063Z" }, - { url = "https://files.pythonhosted.org/packages/b6/f3/8ae2037967e2126689a0c11d99e2b707134a565191e92c60ca2572aec60a/mypy-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8161b6ff4392410023224f0969d17db93e1e154bc3e4ba62598e720723ae211", size = 14045750, upload-time = "2026-05-11T18:31:48.151Z" }, - { url = "https://files.pythonhosted.org/packages/a0/32/615eb5911859e43d054941b0d0a7d06cfa2870eba86529cf385b052b111c/mypy-2.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf03e12003084a67395184d3eb8cbd6a489dc3655b5664b28c210a9e2403ab0b", size = 15061630, upload-time = "2026-05-11T18:37:06.898Z" }, - { url = "https://files.pythonhosted.org/packages/d4/03/4eafbfff8bfab1b87082741eae6e6a624028c984e6708b73bce2a8570c9d/mypy-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:20509760fd791c51579d573153407d226385ec1f8bcce55d730b354f3336bc22", size = 15288831, upload-time = "2026-05-11T18:31:18.07Z" }, - { url = "https://files.pythonhosted.org/packages/99/ee/919661478e5891a3c96e549c036e467e64563ab85995b10c53c8358e16a3/mypy-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:6753d0c1fdd6b1a23b9e4f283ce80b2153b724adcb2653b20b85a8a28ac6436b", size = 11135228, upload-time = "2026-05-11T18:34:31.23Z" }, - { url = "https://files.pythonhosted.org/packages/24/0a/6a12b9782ca0831a553192f351679f4548abc9d19a7cc93bb7feb02084c7/mypy-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:98ebb6589bb3b6d0c6f0c459d53ca55b8091fbc13d277c4041c885392e8195e8", size = 10040684, upload-time = "2026-05-11T18:36:48.199Z" }, - { url = "https://files.pythonhosted.org/packages/6e/dd/c7191469c777f07689c032a8f7326e393ea34c92d6d76eb7ce5ba57ea66d/mypy-2.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:35aac3bb114e03888f535d5eb51b8bafbb3266586b599da1940f9b1be3ec5bd5", size = 14852174, upload-time = "2026-05-11T18:31:38.929Z" }, - { url = "https://files.pythonhosted.org/packages/55/8c/aed55408879043d72bb9135f4d0d19a02b886dd569631e113e3d2706cb8d/mypy-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8de55a8c861f2a49331f807be98d90caeceeef520bde13d43a160207f8af613e", size = 13651542, upload-time = "2026-05-11T18:36:04.636Z" }, - { url = "https://files.pythonhosted.org/packages/3a/8e/f371a824b1f1fa8ea6e3dbb8703d232977d572be2329554a3bc4d960302f/mypy-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5fdf2941a07434af755837d9880f7d7d25f1dacb1af9dcd4b9b66f2220a3024e", size = 14033929, upload-time = "2026-05-11T18:35:55.742Z" }, - { url = "https://files.pythonhosted.org/packages/94/21/f54be870d6dd53a82c674407e0f8eed7174b05ec78d42e5abd7b42e84fd5/mypy-2.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e195b817c13f02352a9c124301f9f30f078405444679b6753c1b96b6eed37285", size = 15039200, upload-time = "2026-05-11T18:33:10.281Z" }, - { url = "https://files.pythonhosted.org/packages/17/99/bf21748626a40ce59fd29a39386ab46afec88b7bd2f0fa6c3a97c995523f/mypy-2.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5431d42af987ebd92ba2f71d45c85ed41d8e6ca9f5fd209a69f68f707d2469e5", size = 15272690, upload-time = "2026-05-11T18:32:07.205Z" }, - { url = "https://files.pythonhosted.org/packages/d6/d7/9e90d2cf47100bea550ed2bc7b0d4de3a62181d84d5e37da0003e8462637/mypy-2.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:767fe8c66dc3e01e19e1737d4c38ebefead16125e1b8e58ad421903b376f5c65", size = 11147435, upload-time = "2026-05-11T18:33:56.477Z" }, - { url = "https://files.pythonhosted.org/packages/ec/46/e5c449e858798e35ffc90946282a27c62a77be743fe17480e4977374eb91/mypy-2.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:ecfe70d43775ab99562ab128ce49854a362044c9f894961f68f898c23cb7429d", size = 10035052, upload-time = "2026-05-11T18:32:30.049Z" }, - { url = "https://files.pythonhosted.org/packages/b0/ca/b279a672e874aedd5498ae25f722dacc8aa86bbffb939b3f97cbb1cf6686/mypy-2.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7354c5a7f69d9345c3d6e69921d57088eea3ddeeb6b20d34c1b3855b02c36ec2", size = 14848422, upload-time = "2026-05-11T18:35:45.984Z" }, - { url = "https://files.pythonhosted.org/packages/27/e6/3efe56c631d959b9b4454e208b0ac4b7f4f58b404c89f8bec7b49efdfc21/mypy-2.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:49890d4f76ac9e06ec117f9e09f3174da70a620a0c300953d8595c926e80947f", size = 13677374, upload-time = "2026-05-11T18:36:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/84/7f/8107ea87a44fd1f1b59882442f033c9c3488c127201b1d1d15f1cbd6022e/mypy-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:761be68e023ef5d94678772396a8af1220030f80837a3afd8d0aef3b419666f4", size = 14055743, upload-time = "2026-05-11T18:35:18.361Z" }, - { url = "https://files.pythonhosted.org/packages/51/4d/b6d34db183133b83761b9199a82d31557cdbb70a380d8c3b3438e11882a3/mypy-2.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c90345fc182dc363b891350457ec69c35140858538f38b4540845afcc32b1aef", size = 15020937, upload-time = "2026-05-11T18:34:59.618Z" }, - { url = "https://files.pythonhosted.org/packages/ff/d7/f08360c691d758acb02f45022c34d98b92892f4ea756644e1000d4b9f3d8/mypy-2.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b84802e7b5a6daf1f5e15bc9fcd7ddae77be13981ffab037f1c67bb84d67d135", size = 15253371, upload-time = "2026-05-11T18:36:41.081Z" }, - { url = "https://files.pythonhosted.org/packages/67/1b/09460a13719530a19bce27bd3bc8449e83569dd2ba7faf51c9c3c30c0b61/mypy-2.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:022c771234936ceac541ebaf836fe9e2abeb3f5e09aff21588fe543ff006fe21", size = 11326429, upload-time = "2026-05-11T18:34:13.526Z" }, - { url = "https://files.pythonhosted.org/packages/40/62/75dbf0f82f7b6680340efc614af29dd0b3c17b8a4f1cd09b8bd2fd6bc814/mypy-2.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:498207db725cec88829a6a5c2fc771205fd043719ef98bc49aba8fb9fc4e6d57", size = 10218799, upload-time = "2026-05-11T18:32:23.491Z" }, - { url = "https://files.pythonhosted.org/packages/b2/66/caca04ed7d972fb6eb6dd1ccd6df1de5c38fae8c5b3dc1c4e8e0d85ee6b9/mypy-2.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7d5e5cad0efeba72b93cd17490cc0d69c5ac9ca132994fe3fb0314808aeeb83e", size = 15923458, upload-time = "2026-05-11T18:35:28.64Z" }, - { url = "https://files.pythonhosted.org/packages/ed/52/2d90cbe49d014b13ed7ff337930c30bad35893fe38a1e4641e756bb62191/mypy-2.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ff715050c127d724fd260a2e666e7747fdd83511c0c47d449d98238970aef780", size = 14757697, upload-time = "2026-05-11T18:36:14.208Z" }, - { url = "https://files.pythonhosted.org/packages/ac/37/d98f4a14e081b238992d0ed96b6d39c7cc0148c9699eb71eaa68629665ea/mypy-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:82208da9e09414d520e912d3e462d454854bed0810b71540bb016dcbca7308fd", size = 15405638, upload-time = "2026-05-11T18:33:48.249Z" }, - { url = "https://files.pythonhosted.org/packages/a3/c2/15c46613b24a84fad2aea1248bf9619b99c2767ae9071fe224c179a0b7d4/mypy-2.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e79ebc1b904b84f0310dff7469655a9c36c7a68bddb37bdd42b67a332df61d08", size = 16215852, upload-time = "2026-05-11T18:32:50.296Z" }, - { url = "https://files.pythonhosted.org/packages/5c/90/9c16a57f482c76d25f6379762b56bbf65c711d8158cf271fb2802cfb0640/mypy-2.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e583edc957cfb0deb142079162ae826f58449b116c1d442f2d91c69d9fced081", size = 16452695, upload-time = "2026-05-11T18:33:38.182Z" }, - { url = "https://files.pythonhosted.org/packages/0f/4c/215a4eeb63cacc5f17f516691ea7285d11e249802b942476bff15922a314/mypy-2.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b33b6cd332695bba180d55e717a79d3038e479a2c49cc5eb3d53603409b9a5d7", size = 12866622, upload-time = "2026-05-11T18:34:39.945Z" }, - { url = "https://files.pythonhosted.org/packages/4b/50/1043e1db5f455ffe4c9ab22747cd8ca2bc492b1e4f4e21b130a44ee2b217/mypy-2.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:4f910fe825376a7b66ef7ca8c98e5a149e8cd64c19ae71d84047a74ee060d4e6", size = 10610798, upload-time = "2026-05-11T18:36:31.444Z" }, - { url = "https://files.pythonhosted.org/packages/0d/2a/13ca1f292f6db1b98ff495ef3467736b331621c5917cad984b7043e7348d/mypy-2.1.0-py3-none-any.whl", hash = "sha256:a663814603a5c563fb87a4f96fb473eeb30d1f5a4885afcf44f9db000a366289", size = 2693302, upload-time = "2026-05-11T18:31:29.246Z" }, + { url = "https://files.pythonhosted.org/packages/76/97/ce2502df2cecf2ef997b6c6527c4a223b92feb9e7b790cdc8dcd683f3a8a/mypy-1.20.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cf5a4db6dca263010e2c7bff081c89383c72d187ba2cf4c44759aac970e2f0c4", size = 14457059, upload-time = "2026-04-21T17:06:14.935Z" }, + { url = "https://files.pythonhosted.org/packages/c9/34/417ee60b822cc80c0f3dc9f495ad7fd8dbb8d8b2cf4baf22d4046d25d01d/mypy-1.20.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b0e817b518bff7facd7f85ea05b643ad8bdcce684cf29784987b0a7c8e1f997", size = 13346816, upload-time = "2026-04-21T17:10:41.433Z" }, + { url = "https://files.pythonhosted.org/packages/4a/85/e20951978702df58379d0bcc2e8f7ccdca4e78cd7dc66dd3ddbf9b29d517/mypy-1.20.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97d7b9a485b40f8ca425460e89bf1da2814625b2da627c0dcc6aa46c92631d14", size = 13772593, upload-time = "2026-04-21T17:08:11.24Z" }, + { url = "https://files.pythonhosted.org/packages/63/a5/5441a13259ec516c56fd5de0fd96a69a9590ae6c5e5d3e5174aa84b97973/mypy-1.20.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e1c12f6d2db3d78b909b5f77513c11eb7f2dd2782b96a3ab6dffc7d44575c99", size = 14656635, upload-time = "2026-04-21T17:09:54.042Z" }, + { url = "https://files.pythonhosted.org/packages/3b/51/b89c69157c5e1f19fd125a65d991166a26906e7902f026f00feebbcfa2b9/mypy-1.20.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89dce27e142d25ffbc154c1819383b69f2e9234dc4ed4766f42e0e8cb264ab5c", size = 14943278, upload-time = "2026-04-21T17:09:15.599Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/6b0eeecfe96d7cce1d71c66b8e03cb304aa70ec11f1955dc1d6b46aca3c3/mypy-1.20.2-cp310-cp310-win_amd64.whl", hash = "sha256:f376e37f9bf2a946872fc5fd1199c99310748e3c26c7a26683f13f8bdb756cbd", size = 10851915, upload-time = "2026-04-21T17:06:03.5Z" }, + { url = "https://files.pythonhosted.org/packages/3c/36/6593dc88545d75fb96416184be5392da5e2a8e8c2802a8597913e16ae25c/mypy-1.20.2-cp310-cp310-win_arm64.whl", hash = "sha256:6e2b469efd811707bc530fd1effef0f5d6eebcb7fe376affae69025da4b979a2", size = 9786676, upload-time = "2026-04-21T17:07:02.035Z" }, + { url = "https://files.pythonhosted.org/packages/1f/4d/9ebeae211caccbdaddde7ed5e31dfcf57faac66be9b11deb1dc6526c8078/mypy-1.20.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4077797a273e56e8843d001e9dfe4ba10e33323d6ade647ff260e5cd97d9758c", size = 14371307, upload-time = "2026-04-21T17:08:56.442Z" }, + { url = "https://files.pythonhosted.org/packages/95/d7/93473d34b61f04fac1aecc01368485c89c5c4af7a4b9a0cab5d77d04b63f/mypy-1.20.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cdecf62abcc4292500d7858aeae87a1f8f1150f4c4dd08fb0b336ee79b2a6df3", size = 13258917, upload-time = "2026-04-21T17:05:50.978Z" }, + { url = "https://files.pythonhosted.org/packages/e2/30/3dd903e8bafb7b5f7bf87fcd58f8382086dea2aa19f0a7b357f21f63071b/mypy-1.20.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c566c3a88b6ece59b3d70f65bedef17304f48eb52ff040a6a18214e1917b3254", size = 13700516, upload-time = "2026-04-21T17:11:33.161Z" }, + { url = "https://files.pythonhosted.org/packages/07/05/c61a140aba4c729ac7bc99ae26fc627c78a6e08f5b9dd319244ea71a3d7e/mypy-1.20.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0deb80d062b2479f2c87ae568f89845afc71d11bc41b04179e58165fd9f31e98", size = 14562889, upload-time = "2026-04-21T17:05:27.674Z" }, + { url = "https://files.pythonhosted.org/packages/fd/87/da78243742ffa8a36d98c3010f0d829f93d5da4e6786f1a1a6f2ad616502/mypy-1.20.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bba9ad231e92a3e424b3e56b65aa17704993425bba97e302c832f9466bb85bac", size = 14803844, upload-time = "2026-04-21T17:10:06.2Z" }, + { url = "https://files.pythonhosted.org/packages/37/52/10a1ddf91b40f843943a3c6db51e2df59c9e237f29d355e95eaab427461f/mypy-1.20.2-cp311-cp311-win_amd64.whl", hash = "sha256:baf593f2765fa3a6b1ef95807dbaa3d25b594f6a52adcc506a6b9cb115e1be67", size = 10846300, upload-time = "2026-04-21T17:12:23.886Z" }, + { url = "https://files.pythonhosted.org/packages/20/02/f9a4415b664c53bd34d6709be59da303abcae986dc4ac847b402edb6fa1e/mypy-1.20.2-cp311-cp311-win_arm64.whl", hash = "sha256:20175a1c0f49863946ec20b7f63255768058ac4f07d2b9ded6a6b46cfb5a9100", size = 9779498, upload-time = "2026-04-21T17:09:23.695Z" }, + { url = "https://files.pythonhosted.org/packages/71/4e/7560e4528db9e9b147e4c0f22660466bf30a0a1fe3d63d1b9d3b0fd354ee/mypy-1.20.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4dbfcf869f6b0517f70cf0030ba6ea1d6645e132337a7d5204a18d8d5636c02b", size = 14539393, upload-time = "2026-04-21T17:07:12.52Z" }, + { url = "https://files.pythonhosted.org/packages/32/d9/34a5efed8124f5a9234f55ac6a4ced4201e2c5b81e1109c49ad23190ec8c/mypy-1.20.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b6481b228d072315b053210b01ac320e1be243dc17f9e5887ef167f23f5fae4", size = 13361642, upload-time = "2026-04-21T17:06:53.742Z" }, + { url = "https://files.pythonhosted.org/packages/d1/14/eb377acf78c03c92d566a1510cda8137348215b5335085ef662ab82ecd3a/mypy-1.20.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34397cdced6b90b836e38182076049fdb41424322e0b0728c946b0939ebdf9f6", size = 13740347, upload-time = "2026-04-21T17:12:04.73Z" }, + { url = "https://files.pythonhosted.org/packages/b9/94/7e4634a32b641aa1c112422eed1bbece61ee16205f674190e8b536f884de/mypy-1.20.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5da6976f20cae27059ea8d0c86e7cef3de720e04c4bb9ee18e3690fdb792066", size = 14734042, upload-time = "2026-04-21T17:07:43.16Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f3/f7e62395cb7f434541b4491a01149a4439e28ace4c0c632bbf5431e92d1f/mypy-1.20.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:56908d7e08318d39f85b1f0c6cfd47b0cac1a130da677630dac0de3e0623e102", size = 14964958, upload-time = "2026-04-21T17:11:00.665Z" }, + { url = "https://files.pythonhosted.org/packages/3e/0d/47e3c3a0ec2a876e35aeac365df3cac7776c36bbd4ed18cc521e1b9d255b/mypy-1.20.2-cp312-cp312-win_amd64.whl", hash = "sha256:d52ad8d78522da1d308789df651ee5379088e77c76cb1994858d40a426b343b9", size = 10911340, upload-time = "2026-04-21T17:10:49.179Z" }, + { url = "https://files.pythonhosted.org/packages/d6/b2/6c852d72e0ea8b01f49da817fb52539993cde327e7d010e0103dc12d0dac/mypy-1.20.2-cp312-cp312-win_arm64.whl", hash = "sha256:785b08db19c9f214dc37d65f7c165d19a30fcecb48abfa30f31b01b5acaabb58", size = 9833947, upload-time = "2026-04-21T17:09:05.267Z" }, + { url = "https://files.pythonhosted.org/packages/5b/c4/b93812d3a192c9bcf5df405bd2f30277cd0e48106a14d1023c7f6ed6e39b/mypy-1.20.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:edfbfca868cdd6bd8d974a60f8a3682f5565d3f5c99b327640cedd24c4264026", size = 14524670, upload-time = "2026-04-21T17:10:30.737Z" }, + { url = "https://files.pythonhosted.org/packages/f3/47/42c122501bff18eaf1e8f457f5c017933452d8acdc52918a9f59f6812955/mypy-1.20.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e2877a02380adfcdbc69071a0f74d6e9dbbf593c0dc9d174e1f223ffd5281943", size = 13336218, upload-time = "2026-04-21T17:08:44.069Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/75bbc92f41725fbd585fb17b440b1119b576105df1013622983e18640a93/mypy-1.20.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7488448de6007cd5177c6cea0517ac33b4c0f5ee9b5e9f2be51ce75511a85517", size = 13724906, upload-time = "2026-04-21T17:08:01.02Z" }, + { url = "https://files.pythonhosted.org/packages/a1/32/4c49da27a606167391ff0c39aa955707a00edc500572e562f7c36c08a71f/mypy-1.20.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bb9c2fa06887e21d6a3a868762acb82aec34e2c6fd0174064f27c93ede68ad15", size = 14726046, upload-time = "2026-04-21T17:11:22.354Z" }, + { url = "https://files.pythonhosted.org/packages/7f/fc/4e354a1bd70216359deb0c9c54847ee6b32ef78dfb09f5131ff99b494078/mypy-1.20.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d56a78b646f2e3daa865bc70cd5ec5a46c50045801ca8ff17a0c43abc97e3ee", size = 14955587, upload-time = "2026-04-21T17:12:16.033Z" }, + { url = "https://files.pythonhosted.org/packages/62/b2/c0f2056e9eb8f08c62cafd9715e4584b89132bdc832fcf85d27d07b5f3e5/mypy-1.20.2-cp313-cp313-win_amd64.whl", hash = "sha256:2a4102b03bb7481d9a91a6da8d174740c9c8c4401024684b9ca3b7cc5e49852f", size = 10922681, upload-time = "2026-04-21T17:06:35.842Z" }, + { url = "https://files.pythonhosted.org/packages/e5/14/065e333721f05de8ef683d0aa804c23026bcc287446b61cac657b902ccac/mypy-1.20.2-cp313-cp313-win_arm64.whl", hash = "sha256:a95a9248b0c6fd933a442c03c3b113c3b61320086b88e2c444676d3fd1ca3330", size = 9830560, upload-time = "2026-04-21T17:07:51.023Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d1/b4ec96b0ecc620a4443570c6e95c867903428cfcde4206518eafdd5880c3/mypy-1.20.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:419413398fe250aae057fd2fe50166b61077083c9b82754c341cf4fd73038f30", size = 14524561, upload-time = "2026-04-21T17:06:27.325Z" }, + { url = "https://files.pythonhosted.org/packages/3a/63/d2c2ff4fa66bc49477d32dfa26e8a167ba803ea6a69c5efb416036909d30/mypy-1.20.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e73c07f23009962885c197ccb9b41356a30cc0e5a1d0c2ea8fd8fb1362d7f924", size = 13363883, upload-time = "2026-04-21T17:11:11.239Z" }, + { url = "https://files.pythonhosted.org/packages/2a/56/983916806bf4eddeaaa2c9230903c3669c6718552a921154e1c5182c701f/mypy-1.20.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c64e5973df366b747646fc98da921f9d6eba9716d57d1db94a83c026a08e0fb", size = 13742945, upload-time = "2026-04-21T17:08:34.181Z" }, + { url = "https://files.pythonhosted.org/packages/19/65/0cd9285ab010ee8214c83d67c6b49417c40d86ce46f1aa109457b5a9b8d7/mypy-1.20.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a65aa591af023864fd08a97da9974e919452cfe19cb146c8a5dc692626445dc", size = 14706163, upload-time = "2026-04-21T17:05:15.51Z" }, + { url = "https://files.pythonhosted.org/packages/94/97/48ff3b297cafcc94d185243a9190836fb1b01c1b0918fff64e941e973cc9/mypy-1.20.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4fef51b01e638974a6e69885687e9bd40c8d1e09a6cd291cca0619625cf1f558", size = 14938677, upload-time = "2026-04-21T17:05:39.562Z" }, + { url = "https://files.pythonhosted.org/packages/fd/a1/1b4233d255bdd0b38a1f284feeb1c143ca508c19184964e22f8d837ec851/mypy-1.20.2-cp314-cp314-win_amd64.whl", hash = "sha256:913485a03f1bcf5d279409a9d2b9ed565c151f61c09f29991e5faa14033da4c8", size = 11089322, upload-time = "2026-04-21T17:06:44.29Z" }, + { url = "https://files.pythonhosted.org/packages/78/c2/ce7ee2ba36aeb954ba50f18fa25d9c1188578654b97d02a66a15b6f09531/mypy-1.20.2-cp314-cp314-win_arm64.whl", hash = "sha256:c3bae4f855d965b5453784300c12ffc63a548304ac7f99e55d4dc7c898673aa3", size = 10017775, upload-time = "2026-04-21T17:07:20.732Z" }, + { url = "https://files.pythonhosted.org/packages/4e/a1/9d93a7d0b5859af0ead82b4888b46df6c8797e1bc5e1e262a08518c6d48e/mypy-1.20.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2de3dcea53babc1c3237a19002bc3d228ce1833278f093b8d619e06e7cc79609", size = 15549002, upload-time = "2026-04-21T17:08:23.107Z" }, + { url = "https://files.pythonhosted.org/packages/00/d2/09a6a10ee1bf0008f6c144d9676f2ca6a12512151b4e0ad0ff6c4fac5337/mypy-1.20.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:52b176444e2e5054dfcbcb8c75b0b719865c96247b37407184bbfca5c353f2c2", size = 14401942, upload-time = "2026-04-21T17:07:31.837Z" }, + { url = "https://files.pythonhosted.org/packages/57/da/9594b75c3c019e805250bed3583bdf4443ff9e6ef08f97e39ae308cb06f2/mypy-1.20.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:688c3312e5dadb573a2c69c82af3a298d43ecf9e6d264e0f95df960b5f6ac19c", size = 15041649, upload-time = "2026-04-21T17:09:34.653Z" }, + { url = "https://files.pythonhosted.org/packages/97/77/f75a65c278e6e8eba2071f7f5a90481891053ecc39878cc444634d892abe/mypy-1.20.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29752dbbf8cc53f89f6ac096d363314333045c257c9c75cbd189ca2de0455744", size = 15864588, upload-time = "2026-04-21T17:11:44.936Z" }, + { url = "https://files.pythonhosted.org/packages/d7/46/1a4e1c66e96c1a3246ddf5403d122ac9b0a8d2b7e65730b9d6533ba7a6d3/mypy-1.20.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:803203d2b6ea644982c644895c2f78b28d0e208bba7b27d9b921e0ec5eb207c6", size = 16093956, upload-time = "2026-04-21T17:10:17.683Z" }, + { url = "https://files.pythonhosted.org/packages/5a/2c/78a8851264dec38cd736ca5b8bc9380674df0dd0be7792f538916157716c/mypy-1.20.2-cp314-cp314t-win_amd64.whl", hash = "sha256:9bcb8aa397ff0093c824182fd76a935a9ba7ad097fcbef80ae89bf6c1731d8ec", size = 12568661, upload-time = "2026-04-21T17:11:54.473Z" }, + { url = "https://files.pythonhosted.org/packages/83/01/cd7318aa03493322ce275a0e14f4f52b8896335e4e79d4fb8153a7ad2b77/mypy-1.20.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e061b58443f1736f8a37c48978d7ab581636d6ab03e3d4f99e3fa90463bb9382", size = 10389240, upload-time = "2026-04-21T17:09:42.719Z" }, + { url = "https://files.pythonhosted.org/packages/28/9a/f23c163e25b11074188251b0b5a0342625fc1cdb6af604757174fa9acc9b/mypy-1.20.2-py3-none-any.whl", hash = "sha256:a94c5a76ab46c5e6257c7972b6c8cff0574201ca7dc05647e33e795d78680563", size = 2637314, upload-time = "2026-04-21T17:05:54.5Z" }, ] [[package]] @@ -752,85 +596,85 @@ wheels = [ [[package]] name = "numpy" -version = "2.4.6" +version = "2.4.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.15'", "python_full_version >= '3.11' and python_full_version < '3.15'", ] -sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/9f/b8cef5bffa569759033adda9481211426f12f53299629b410340795c2514/numpy-2.4.4.tar.gz", hash = "sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0", size = 20731587, upload-time = "2026-03-29T13:22:01.298Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" }, - { url = "https://files.pythonhosted.org/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" }, - { url = "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" }, - { url = "https://files.pythonhosted.org/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" }, - { url = "https://files.pythonhosted.org/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" }, - { url = "https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" }, - { url = "https://files.pythonhosted.org/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" }, - { url = "https://files.pythonhosted.org/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" }, - { url = "https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" }, - { url = "https://files.pythonhosted.org/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" }, - { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, - { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, - { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, - { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, - { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, - { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, - { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, - { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, - { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, - { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, - { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, - { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, - { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, - { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, - { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, - { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, - { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, - { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, - { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, - { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, - { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, - { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, - { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, - { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, - { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, - { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, - { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, - { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, - { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, - { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, - { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, - { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, - { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, - { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, - { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, - { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, - { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, - { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, - { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, - { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, - { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, - { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, - { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, - { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, - { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, - { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, - { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, - { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, - { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, - { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, - { url = "https://files.pythonhosted.org/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" }, - { url = "https://files.pythonhosted.org/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" }, - { url = "https://files.pythonhosted.org/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" }, - { url = "https://files.pythonhosted.org/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" }, - { url = "https://files.pythonhosted.org/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" }, - { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, + { url = "https://files.pythonhosted.org/packages/ef/c6/4218570d8c8ecc9704b5157a3348e486e84ef4be0ed3e38218ab473c83d2/numpy-2.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f983334aea213c99992053ede6168500e5f086ce74fbc4acc3f2b00f5762e9db", size = 16976799, upload-time = "2026-03-29T13:18:15.438Z" }, + { url = "https://files.pythonhosted.org/packages/dd/92/b4d922c4a5f5dab9ed44e6153908a5c665b71acf183a83b93b690996e39b/numpy-2.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:72944b19f2324114e9dc86a159787333b77874143efcf89a5167ef83cfee8af0", size = 14971552, upload-time = "2026-03-29T13:18:18.606Z" }, + { url = "https://files.pythonhosted.org/packages/8a/dc/df98c095978fa6ee7b9a9387d1d58cbb3d232d0e69ad169a4ce784bde4fd/numpy-2.4.4-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:86b6f55f5a352b48d7fbfd2dbc3d5b780b2d79f4d3c121f33eb6efb22e9a2015", size = 5476566, upload-time = "2026-03-29T13:18:21.532Z" }, + { url = "https://files.pythonhosted.org/packages/28/34/b3fdcec6e725409223dd27356bdf5a3c2cc2282e428218ecc9cb7acc9763/numpy-2.4.4-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:ba1f4fc670ed79f876f70082eff4f9583c15fb9a4b89d6188412de4d18ae2f40", size = 6806482, upload-time = "2026-03-29T13:18:23.634Z" }, + { url = "https://files.pythonhosted.org/packages/68/62/63417c13aa35d57bee1337c67446761dc25ea6543130cf868eace6e8157b/numpy-2.4.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a87ec22c87be071b6bdbd27920b129b94f2fc964358ce38f3822635a3e2e03d", size = 15973376, upload-time = "2026-03-29T13:18:26.677Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c5/9fcb7e0e69cef59cf10c746b84f7d58b08bc66a6b7d459783c5a4f6101a6/numpy-2.4.4-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df3775294accfdd75f32c74ae39fcba920c9a378a2fc18a12b6820aa8c1fb502", size = 16925137, upload-time = "2026-03-29T13:18:30.14Z" }, + { url = "https://files.pythonhosted.org/packages/7e/43/80020edacb3f84b9efdd1591120a4296462c23fd8db0dde1666f6ef66f13/numpy-2.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0d4e437e295f18ec29bc79daf55e8a47a9113df44d66f702f02a293d93a2d6dd", size = 17329414, upload-time = "2026-03-29T13:18:33.733Z" }, + { url = "https://files.pythonhosted.org/packages/fd/06/af0658593b18a5f73532d377188b964f239eb0894e664a6c12f484472f97/numpy-2.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6aa3236c78803afbcb255045fbef97a9e25a1f6c9888357d205ddc42f4d6eba5", size = 18658397, upload-time = "2026-03-29T13:18:37.511Z" }, + { url = "https://files.pythonhosted.org/packages/e6/ce/13a09ed65f5d0ce5c7dd0669250374c6e379910f97af2c08c57b0608eee4/numpy-2.4.4-cp311-cp311-win32.whl", hash = "sha256:30caa73029a225b2d40d9fae193e008e24b2026b7ee1a867b7ee8d96ca1a448e", size = 6239499, upload-time = "2026-03-29T13:18:40.372Z" }, + { url = "https://files.pythonhosted.org/packages/bd/63/05d193dbb4b5eec1eca73822d80da98b511f8328ad4ae3ca4caf0f4db91d/numpy-2.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:6bbe4eb67390b0a0265a2c25458f6b90a409d5d069f1041e6aff1e27e3d9a79e", size = 12614257, upload-time = "2026-03-29T13:18:42.95Z" }, + { url = "https://files.pythonhosted.org/packages/87/c5/8168052f080c26fa984c413305012be54741c9d0d74abd7fbeeccae3889f/numpy-2.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:fcfe2045fd2e8f3cb0ce9d4ba6dba6333b8fa05bb8a4939c908cd43322d14c7e", size = 10486775, upload-time = "2026-03-29T13:18:45.835Z" }, + { url = "https://files.pythonhosted.org/packages/28/05/32396bec30fb2263770ee910142f49c1476d08e8ad41abf8403806b520ce/numpy-2.4.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15716cfef24d3a9762e3acdf87e27f58dc823d1348f765bbea6bef8c639bfa1b", size = 16689272, upload-time = "2026-03-29T13:18:49.223Z" }, + { url = "https://files.pythonhosted.org/packages/c5/f3/a983d28637bfcd763a9c7aafdb6d5c0ebf3d487d1e1459ffdb57e2f01117/numpy-2.4.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23cbfd4c17357c81021f21540da84ee282b9c8fba38a03b7b9d09ba6b951421e", size = 14699573, upload-time = "2026-03-29T13:18:52.629Z" }, + { url = "https://files.pythonhosted.org/packages/9b/fd/e5ecca1e78c05106d98028114f5c00d3eddb41207686b2b7de3e477b0e22/numpy-2.4.4-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:8b3b60bb7cba2c8c81837661c488637eee696f59a877788a396d33150c35d842", size = 5204782, upload-time = "2026-03-29T13:18:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/de/2f/702a4594413c1a8632092beae8aba00f1d67947389369b3777aed783fdca/numpy-2.4.4-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:e4a010c27ff6f210ff4c6ef34394cd61470d01014439b192ec22552ee867f2a8", size = 6552038, upload-time = "2026-03-29T13:18:57.769Z" }, + { url = "https://files.pythonhosted.org/packages/7f/37/eed308a8f56cba4d1fdf467a4fc67ef4ff4bf1c888f5fc980481890104b1/numpy-2.4.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f9e75681b59ddaa5e659898085ae0eaea229d054f2ac0c7e563a62205a700121", size = 15670666, upload-time = "2026-03-29T13:19:00.341Z" }, + { url = "https://files.pythonhosted.org/packages/0a/0d/0e3ecece05b7a7e87ab9fb587855548da437a061326fff64a223b6dcb78a/numpy-2.4.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:81f4a14bee47aec54f883e0cad2d73986640c1590eb9bfaaba7ad17394481e6e", size = 16645480, upload-time = "2026-03-29T13:19:03.63Z" }, + { url = "https://files.pythonhosted.org/packages/34/49/f2312c154b82a286758ee2f1743336d50651f8b5195db18cdb63675ff649/numpy-2.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:62d6b0f03b694173f9fcb1fb317f7222fd0b0b103e784c6549f5e53a27718c44", size = 17020036, upload-time = "2026-03-29T13:19:07.428Z" }, + { url = "https://files.pythonhosted.org/packages/7b/e9/736d17bd77f1b0ec4f9901aaec129c00d59f5d84d5e79bba540ef12c2330/numpy-2.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fbc356aae7adf9e6336d336b9c8111d390a05df88f1805573ebb0807bd06fd1d", size = 18368643, upload-time = "2026-03-29T13:19:10.775Z" }, + { url = "https://files.pythonhosted.org/packages/63/f6/d417977c5f519b17c8a5c3bc9e8304b0908b0e21136fe43bf628a1343914/numpy-2.4.4-cp312-cp312-win32.whl", hash = "sha256:0d35aea54ad1d420c812bfa0385c71cd7cc5bcf7c65fed95fc2cd02fe8c79827", size = 5961117, upload-time = "2026-03-29T13:19:13.464Z" }, + { url = "https://files.pythonhosted.org/packages/2d/5b/e1deebf88ff431b01b7406ca3583ab2bbb90972bbe1c568732e49c844f7e/numpy-2.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:b5f0362dc928a6ecd9db58868fca5e48485205e3855957bdedea308f8672ea4a", size = 12320584, upload-time = "2026-03-29T13:19:16.155Z" }, + { url = "https://files.pythonhosted.org/packages/58/89/e4e856ac82a68c3ed64486a544977d0e7bdd18b8da75b78a577ca31c4395/numpy-2.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:846300f379b5b12cc769334464656bc882e0735d27d9726568bc932fdc49d5ec", size = 10221450, upload-time = "2026-03-29T13:19:18.994Z" }, + { url = "https://files.pythonhosted.org/packages/14/1d/d0a583ce4fefcc3308806a749a536c201ed6b5ad6e1322e227ee4848979d/numpy-2.4.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:08f2e31ed5e6f04b118e49821397f12767934cfdd12a1ce86a058f91e004ee50", size = 16684933, upload-time = "2026-03-29T13:19:22.47Z" }, + { url = "https://files.pythonhosted.org/packages/c1/62/2b7a48fbb745d344742c0277f01286dead15f3f68e4f359fbfcf7b48f70f/numpy-2.4.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e823b8b6edc81e747526f70f71a9c0a07ac4e7ad13020aa736bb7c9d67196115", size = 14694532, upload-time = "2026-03-29T13:19:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/e5/87/499737bfba066b4a3bebff24a8f1c5b2dee410b209bc6668c9be692580f0/numpy-2.4.4-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4a19d9dba1a76618dd86b164d608566f393f8ec6ac7c44f0cc879011c45e65af", size = 5199661, upload-time = "2026-03-29T13:19:28.31Z" }, + { url = "https://files.pythonhosted.org/packages/cd/da/464d551604320d1491bc345efed99b4b7034143a85787aab78d5691d5a0e/numpy-2.4.4-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:d2a8490669bfe99a233298348acc2d824d496dee0e66e31b66a6022c2ad74a5c", size = 6547539, upload-time = "2026-03-29T13:19:30.97Z" }, + { url = "https://files.pythonhosted.org/packages/7d/90/8d23e3b0dafd024bf31bdec225b3bb5c2dbfa6912f8a53b8659f21216cbf/numpy-2.4.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45dbed2ab436a9e826e302fcdcbe9133f9b0006e5af7168afb8963a6520da103", size = 15668806, upload-time = "2026-03-29T13:19:33.887Z" }, + { url = "https://files.pythonhosted.org/packages/d1/73/a9d864e42a01896bb5974475438f16086be9ba1f0d19d0bb7a07427c4a8b/numpy-2.4.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c901b15172510173f5cb310eae652908340f8dede90fff9e3bf6c0d8dfd92f83", size = 16632682, upload-time = "2026-03-29T13:19:37.336Z" }, + { url = "https://files.pythonhosted.org/packages/34/fb/14570d65c3bde4e202a031210475ae9cde9b7686a2e7dc97ee67d2833b35/numpy-2.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:99d838547ace2c4aace6c4f76e879ddfe02bb58a80c1549928477862b7a6d6ed", size = 17019810, upload-time = "2026-03-29T13:19:40.963Z" }, + { url = "https://files.pythonhosted.org/packages/8a/77/2ba9d87081fd41f6d640c83f26fb7351e536b7ce6dd9061b6af5904e8e46/numpy-2.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0aec54fd785890ecca25a6003fd9a5aed47ad607bbac5cd64f836ad8666f4959", size = 18357394, upload-time = "2026-03-29T13:19:44.859Z" }, + { url = "https://files.pythonhosted.org/packages/a2/23/52666c9a41708b0853fa3b1a12c90da38c507a3074883823126d4e9d5b30/numpy-2.4.4-cp313-cp313-win32.whl", hash = "sha256:07077278157d02f65c43b1b26a3886bce886f95d20aabd11f87932750dfb14ed", size = 5959556, upload-time = "2026-03-29T13:19:47.661Z" }, + { url = "https://files.pythonhosted.org/packages/57/fb/48649b4971cde70d817cf97a2a2fdc0b4d8308569f1dd2f2611959d2e0cf/numpy-2.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:5c70f1cc1c4efbe316a572e2d8b9b9cc44e89b95f79ca3331553fbb63716e2bf", size = 12317311, upload-time = "2026-03-29T13:19:50.67Z" }, + { url = "https://files.pythonhosted.org/packages/ba/d8/11490cddd564eb4de97b4579ef6bfe6a736cc07e94c1598590ae25415e01/numpy-2.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:ef4059d6e5152fa1a39f888e344c73fdc926e1b2dd58c771d67b0acfbf2aa67d", size = 10222060, upload-time = "2026-03-29T13:19:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/99/5d/dab4339177a905aad3e2221c915b35202f1ec30d750dd2e5e9d9a72b804b/numpy-2.4.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4bbc7f303d125971f60ec0aaad5e12c62d0d2c925f0ab1273debd0e4ba37aba5", size = 14822302, upload-time = "2026-03-29T13:19:57.585Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e4/0564a65e7d3d97562ed6f9b0fd0fb0a6f559ee444092f105938b50043876/numpy-2.4.4-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:4d6d57903571f86180eb98f8f0c839fa9ebbfb031356d87f1361be91e433f5b7", size = 5327407, upload-time = "2026-03-29T13:20:00.601Z" }, + { url = "https://files.pythonhosted.org/packages/29/8d/35a3a6ce5ad371afa58b4700f1c820f8f279948cca32524e0a695b0ded83/numpy-2.4.4-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:4636de7fd195197b7535f231b5de9e4b36d2c440b6e566d2e4e4746e6af0ca93", size = 6647631, upload-time = "2026-03-29T13:20:02.855Z" }, + { url = "https://files.pythonhosted.org/packages/f4/da/477731acbd5a58a946c736edfdabb2ac5b34c3d08d1ba1a7b437fa0884df/numpy-2.4.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ad2e2ef14e0b04e544ea2fa0a36463f847f113d314aa02e5b402fdf910ef309e", size = 15727691, upload-time = "2026-03-29T13:20:06.004Z" }, + { url = "https://files.pythonhosted.org/packages/e6/db/338535d9b152beabeb511579598418ba0212ce77cf9718edd70262cc4370/numpy-2.4.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a285b3b96f951841799528cd1f4f01cd70e7e0204b4abebac9463eecfcf2a40", size = 16681241, upload-time = "2026-03-29T13:20:09.417Z" }, + { url = "https://files.pythonhosted.org/packages/e2/a9/ad248e8f58beb7a0219b413c9c7d8151c5d285f7f946c3e26695bdbbe2df/numpy-2.4.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f8474c4241bc18b750be2abea9d7a9ec84f46ef861dbacf86a4f6e043401f79e", size = 17085767, upload-time = "2026-03-29T13:20:13.126Z" }, + { url = "https://files.pythonhosted.org/packages/b5/1a/3b88ccd3694681356f70da841630e4725a7264d6a885c8d442a697e1146b/numpy-2.4.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4e874c976154687c1f71715b034739b45c7711bec81db01914770373d125e392", size = 18403169, upload-time = "2026-03-29T13:20:17.096Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c9/fcfd5d0639222c6eac7f304829b04892ef51c96a75d479214d77e3ce6e33/numpy-2.4.4-cp313-cp313t-win32.whl", hash = "sha256:9c585a1790d5436a5374bac930dad6ed244c046ed91b2b2a3634eb2971d21008", size = 6083477, upload-time = "2026-03-29T13:20:20.195Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e3/3938a61d1c538aaec8ed6fd6323f57b0c2d2d2219512434c5c878db76553/numpy-2.4.4-cp313-cp313t-win_amd64.whl", hash = "sha256:93e15038125dc1e5345d9b5b68aa7f996ec33b98118d18c6ca0d0b7d6198b7e8", size = 12457487, upload-time = "2026-03-29T13:20:22.946Z" }, + { url = "https://files.pythonhosted.org/packages/97/6a/7e345032cc60501721ef94e0e30b60f6b0bd601f9174ebd36389a2b86d40/numpy-2.4.4-cp313-cp313t-win_arm64.whl", hash = "sha256:0dfd3f9d3adbe2920b68b5cd3d51444e13a10792ec7154cd0a2f6e74d4ab3233", size = 10292002, upload-time = "2026-03-29T13:20:25.909Z" }, + { url = "https://files.pythonhosted.org/packages/6e/06/c54062f85f673dd5c04cbe2f14c3acb8c8b95e3384869bb8cc9bff8cb9df/numpy-2.4.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f169b9a863d34f5d11b8698ead99febeaa17a13ca044961aa8e2662a6c7766a0", size = 16684353, upload-time = "2026-03-29T13:20:29.504Z" }, + { url = "https://files.pythonhosted.org/packages/4c/39/8a320264a84404c74cc7e79715de85d6130fa07a0898f67fb5cd5bd79908/numpy-2.4.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2483e4584a1cb3092da4470b38866634bafb223cbcd551ee047633fd2584599a", size = 14704914, upload-time = "2026-03-29T13:20:33.547Z" }, + { url = "https://files.pythonhosted.org/packages/91/fb/287076b2614e1d1044235f50f03748f31fa287e3dbe6abeb35cdfa351eca/numpy-2.4.4-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:2d19e6e2095506d1736b7d80595e0f252d76b89f5e715c35e06e937679ea7d7a", size = 5210005, upload-time = "2026-03-29T13:20:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/63/eb/fcc338595309910de6ecabfcef2419a9ce24399680bfb149421fa2df1280/numpy-2.4.4-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:6a246d5914aa1c820c9443ddcee9c02bec3e203b0c080349533fae17727dfd1b", size = 6544974, upload-time = "2026-03-29T13:20:39.014Z" }, + { url = "https://files.pythonhosted.org/packages/44/5d/e7e9044032a716cdfaa3fba27a8e874bf1c5f1912a1ddd4ed071bf8a14a6/numpy-2.4.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:989824e9faf85f96ec9c7761cd8d29c531ad857bfa1daa930cba85baaecf1a9a", size = 15684591, upload-time = "2026-03-29T13:20:42.146Z" }, + { url = "https://files.pythonhosted.org/packages/98/7c/21252050676612625449b4807d6b695b9ce8a7c9e1c197ee6216c8a65c7c/numpy-2.4.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27a8d92cd10f1382a67d7cf4db7ce18341b66438bdd9f691d7b0e48d104c2a9d", size = 16637700, upload-time = "2026-03-29T13:20:46.204Z" }, + { url = "https://files.pythonhosted.org/packages/b1/29/56d2bbef9465db24ef25393383d761a1af4f446a1df9b8cded4fe3a5a5d7/numpy-2.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e44319a2953c738205bf3354537979eaa3998ed673395b964c1176083dd46252", size = 17035781, upload-time = "2026-03-29T13:20:50.242Z" }, + { url = "https://files.pythonhosted.org/packages/e3/2b/a35a6d7589d21f44cea7d0a98de5ddcbb3d421b2622a5c96b1edf18707c3/numpy-2.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e892aff75639bbef0d2a2cfd55535510df26ff92f63c92cd84ef8d4ba5a5557f", size = 18362959, upload-time = "2026-03-29T13:20:54.019Z" }, + { url = "https://files.pythonhosted.org/packages/64/c9/d52ec581f2390e0f5f85cbfd80fb83d965fc15e9f0e1aec2195faa142cde/numpy-2.4.4-cp314-cp314-win32.whl", hash = "sha256:1378871da56ca8943c2ba674530924bb8ca40cd228358a3b5f302ad60cf875fc", size = 6008768, upload-time = "2026-03-29T13:20:56.912Z" }, + { url = "https://files.pythonhosted.org/packages/fa/22/4cc31a62a6c7b74a8730e31a4274c5dc80e005751e277a2ce38e675e4923/numpy-2.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:715d1c092715954784bc79e1174fc2a90093dc4dc84ea15eb14dad8abdcdeb74", size = 12449181, upload-time = "2026-03-29T13:20:59.548Z" }, + { url = "https://files.pythonhosted.org/packages/70/2e/14cda6f4d8e396c612d1bf97f22958e92148801d7e4f110cabebdc0eef4b/numpy-2.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:2c194dd721e54ecad9ad387c1d35e63dce5c4450c6dc7dd5611283dda239aabb", size = 10496035, upload-time = "2026-03-29T13:21:02.524Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e8/8fed8c8d848d7ecea092dc3469643f9d10bc3a134a815a3b033da1d2039b/numpy-2.4.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2aa0613a5177c264ff5921051a5719d20095ea586ca88cc802c5c218d1c67d3e", size = 14824958, upload-time = "2026-03-29T13:21:05.671Z" }, + { url = "https://files.pythonhosted.org/packages/05/1a/d8007a5138c179c2bf33ef44503e83d70434d2642877ee8fbb230e7c0548/numpy-2.4.4-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:42c16925aa5a02362f986765f9ebabf20de75cdefdca827d14315c568dcab113", size = 5330020, upload-time = "2026-03-29T13:21:08.635Z" }, + { url = "https://files.pythonhosted.org/packages/99/64/ffb99ac6ae93faf117bcbd5c7ba48a7f45364a33e8e458545d3633615dda/numpy-2.4.4-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:874f200b2a981c647340f841730fc3a2b54c9d940566a3c4149099591e2c4c3d", size = 6650758, upload-time = "2026-03-29T13:21:10.949Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6e/795cc078b78a384052e73b2f6281ff7a700e9bf53bcce2ee579d4f6dd879/numpy-2.4.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9b39d38a9bd2ae1becd7eac1303d031c5c110ad31f2b319c6e7d98b135c934d", size = 15729948, upload-time = "2026-03-29T13:21:14.047Z" }, + { url = "https://files.pythonhosted.org/packages/5f/86/2acbda8cc2af5f3d7bfc791192863b9e3e19674da7b5e533fded124d1299/numpy-2.4.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b268594bccac7d7cf5844c7732e3f20c50921d94e36d7ec9b79e9857694b1b2f", size = 16679325, upload-time = "2026-03-29T13:21:17.561Z" }, + { url = "https://files.pythonhosted.org/packages/bc/59/cafd83018f4aa55e0ac6fa92aa066c0a1877b77a615ceff1711c260ffae8/numpy-2.4.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ac6b31e35612a26483e20750126d30d0941f949426974cace8e6b5c58a3657b0", size = 17084883, upload-time = "2026-03-29T13:21:21.106Z" }, + { url = "https://files.pythonhosted.org/packages/f0/85/a42548db84e65ece46ab2caea3d3f78b416a47af387fcbb47ec28e660dc2/numpy-2.4.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8e3ed142f2728df44263aaf5fb1f5b0b99f4070c553a0d7f033be65338329150", size = 18403474, upload-time = "2026-03-29T13:21:24.828Z" }, + { url = "https://files.pythonhosted.org/packages/ed/ad/483d9e262f4b831000062e5d8a45e342166ec8aaa1195264982bca267e62/numpy-2.4.4-cp314-cp314t-win32.whl", hash = "sha256:dddbbd259598d7240b18c9d87c56a9d2fb3b02fe266f49a7c101532e78c1d871", size = 6155500, upload-time = "2026-03-29T13:21:28.205Z" }, + { url = "https://files.pythonhosted.org/packages/c7/03/2fc4e14c7bd4ff2964b74ba90ecb8552540b6315f201df70f137faa5c589/numpy-2.4.4-cp314-cp314t-win_amd64.whl", hash = "sha256:a7164afb23be6e37ad90b2f10426149fd75aee07ca55653d2aa41e66c4ef697e", size = 12637755, upload-time = "2026-03-29T13:21:31.107Z" }, + { url = "https://files.pythonhosted.org/packages/58/78/548fb8e07b1a341746bfbecb32f2c268470f45fa028aacdbd10d9bc73aab/numpy-2.4.4-cp314-cp314t-win_arm64.whl", hash = "sha256:ba203255017337d39f89bdd58417f03c4426f12beed0440cfd933cb15f8669c7", size = 10566643, upload-time = "2026-03-29T13:21:34.339Z" }, + { url = "https://files.pythonhosted.org/packages/6b/33/8fae8f964a4f63ed528264ddf25d2b683d0b663e3cba26961eb838a7c1bd/numpy-2.4.4-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:58c8b5929fcb8287cbd6f0a3fae19c6e03a5c48402ae792962ac465224a629a4", size = 16854491, upload-time = "2026-03-29T13:21:38.03Z" }, + { url = "https://files.pythonhosted.org/packages/bc/d0/1aabee441380b981cf8cdda3ae7a46aa827d1b5a8cce84d14598bc94d6d9/numpy-2.4.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:eea7ac5d2dce4189771cedb559c738a71512768210dc4e4753b107a2048b3d0e", size = 14895830, upload-time = "2026-03-29T13:21:41.509Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b8/aafb0d1065416894fccf4df6b49ef22b8db045187949545bced89c034b8e/numpy-2.4.4-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:51fc224f7ca4d92656d5a5eb315f12eb5fe2c97a66249aa7b5f562528a3be38c", size = 5400927, upload-time = "2026-03-29T13:21:44.747Z" }, + { url = "https://files.pythonhosted.org/packages/d6/77/063baa20b08b431038c7f9ff5435540c7b7265c78cf56012a483019ca72d/numpy-2.4.4-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:28a650663f7314afc3e6ec620f44f333c386aad9f6fc472030865dc0ebb26ee3", size = 6715557, upload-time = "2026-03-29T13:21:47.406Z" }, + { url = "https://files.pythonhosted.org/packages/c7/a8/379542d45a14f149444c5c4c4e7714707239ce9cc1de8c2803958889da14/numpy-2.4.4-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19710a9ca9992d7174e9c52f643d4272dcd1558c5f7af7f6f8190f633bd651a7", size = 15804253, upload-time = "2026-03-29T13:21:50.753Z" }, + { url = "https://files.pythonhosted.org/packages/a2/c8/f0a45426d6d21e7ea3310a15cf90c43a14d9232c31a837702dba437f3373/numpy-2.4.4-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b2aec6af35c113b05695ebb5749a787acd63cafc83086a05771d1e1cd1e555f", size = 16753552, upload-time = "2026-03-29T13:21:54.344Z" }, + { url = "https://files.pythonhosted.org/packages/04/74/f4c001f4714c3ad9ce037e18cf2b9c64871a84951eaa0baf683a9ca9301c/numpy-2.4.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f2cf083b324a467e1ab358c105f6cad5ea950f50524668a80c486ff1db24e119", size = 12509075, upload-time = "2026-03-29T13:21:57.644Z" }, ] [[package]] @@ -998,6 +842,17 @@ dependencies = [ { name = "overture-schema-common" }, { name = "overture-schema-system" }, { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, +] + +[package.dev-dependencies] +test = [ + { name = "overture-schema-addresses-theme" }, + { name = "overture-schema-base-theme" }, + { name = "overture-schema-buildings-theme" }, + { name = "overture-schema-divisions-theme" }, + { name = "overture-schema-places-theme" }, + { name = "overture-schema-transportation-theme" }, ] [package.metadata] @@ -1008,6 +863,17 @@ requires-dist = [ { name = "overture-schema-common", editable = "packages/overture-schema-common" }, { name = "overture-schema-system", editable = "packages/overture-schema-system" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0" }, + { name = "typing-extensions", specifier = ">=4.0" }, +] + +[package.metadata.requires-dev] +test = [ + { name = "overture-schema-addresses-theme", editable = "packages/overture-schema-addresses-theme" }, + { name = "overture-schema-base-theme", editable = "packages/overture-schema-base-theme" }, + { name = "overture-schema-buildings-theme", editable = "packages/overture-schema-buildings-theme" }, + { name = "overture-schema-divisions-theme", editable = "packages/overture-schema-divisions-theme" }, + { name = "overture-schema-places-theme", editable = "packages/overture-schema-places-theme" }, + { name = "overture-schema-transportation-theme", editable = "packages/overture-schema-transportation-theme" }, ] [[package]] @@ -1072,6 +938,22 @@ requires-dist = [ { name = "pydantic", extras = ["email"], specifier = ">=2.12.0" }, ] +[[package]] +name = "overture-schema-pyspark" +source = { editable = "packages/overture-schema-pyspark" } +dependencies = [ + { name = "click" }, + { name = "overture-schema-system" }, + { name = "pyspark" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.0" }, + { name = "overture-schema-system", editable = "packages/overture-schema-system" }, + { name = "pyspark", specifier = ">=3.4" }, +] + [[package]] name = "overture-schema-system" source = { editable = "packages/overture-schema-system" } @@ -1128,6 +1010,7 @@ dev = [ { name = "pydocstyle" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "pytest-testmon" }, { name = "ruff" }, ] @@ -1140,6 +1023,7 @@ dev = [ { name = "pydocstyle", specifier = ">=6.3.0" }, { name = "pytest", specifier = ">=9.0.0" }, { name = "pytest-cov", specifier = ">=7.0.0" }, + { name = "pytest-testmon", specifier = ">=2.2.0" }, { name = "ruff", specifier = ">=0.13.0" }, ] @@ -1185,9 +1069,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "py4j" +version = "0.10.9.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/31/0b210511177070c8d5d3059556194352e5753602fa64b85b7ab81ec1a009/py4j-0.10.9.9.tar.gz", hash = "sha256:f694cad19efa5bd1dee4f3e5270eb406613c974394035e5bfc4ec1aba870b879", size = 761089, upload-time = "2025-01-15T03:53:18.624Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/db/ea0203e495be491c85af87b66e37acfd3bf756fd985f87e46fc5e3bf022c/py4j-0.10.9.9-py2.py3-none-any.whl", hash = "sha256:c7c26e4158defb37b0bb124933163641a2ff6e3a3913f7811b0ddbe07ed61533", size = 203008, upload-time = "2025-01-15T03:53:15.648Z" }, +] + [[package]] name = "pydantic" -version = "2.13.4" +version = "2.13.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -1195,9 +1088,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/e4/40d09941a2cebcb20609b86a559817d5b9291c49dd6f8c87e5feffbe703a/pydantic-2.13.3.tar.gz", hash = "sha256:af09e9d1d09f4e7fe37145c1f577e1d61ceb9a41924bf0094a36506285d0a84d", size = 844068, upload-time = "2026-04-20T14:46:43.632Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, + { url = "https://files.pythonhosted.org/packages/f3/0a/fd7d723f8f8153418fb40cf9c940e82004fce7e987026b08a68a36dd3fe7/pydantic-2.13.3-py3-none-any.whl", hash = "sha256:6db14ac8dfc9a1e57f87ea2c0de670c251240f43cb0c30a5130e9720dc612927", size = 471981, upload-time = "2026-04-20T14:46:41.402Z" }, ] [package.optional-dependencies] @@ -1207,118 +1100,118 @@ email = [ [[package]] name = "pydantic-core" -version = "2.46.4" +version = "2.46.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2a/ef/f7abb56c49382a246fd2ce9c799691e3c3e7175ec74b14d99e798bcddb1a/pydantic_core-2.46.3.tar.gz", hash = "sha256:41c178f65b8c29807239d47e6050262eb6bf84eb695e41101e62e38df4a5bc2c", size = 471412, upload-time = "2026-04-20T14:40:56.672Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/08/f1ba952f1c8ae5581c70fa9c6da89f247b83e3dd8c09c035d5d7931fc23d/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", size = 2113146, upload-time = "2026-05-06T13:37:36.537Z" }, - { url = "https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", size = 1949769, upload-time = "2026-05-06T13:37:46.365Z" }, - { url = "https://files.pythonhosted.org/packages/64/ba/bfb1d928fd5b49e1258935ff104ae356e9fd89384a55bf9f847e9193ad40/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", size = 1974958, upload-time = "2026-05-06T13:37:28.611Z" }, - { url = "https://files.pythonhosted.org/packages/4e/74/76223bfb117b64af743c9b6670d1364516f5c0604f96b48f3272f6af6cc6/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", size = 2042118, upload-time = "2026-05-06T13:36:55.216Z" }, - { url = "https://files.pythonhosted.org/packages/cb/7b/848732968bc8f48f3187542f08358b9d842db564147b256669426ebb1652/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", size = 2222876, upload-time = "2026-05-06T13:38:25.455Z" }, - { url = "https://files.pythonhosted.org/packages/b5/2f/e90b63ee2e14bd8d3db8f705a6d75d64e6ee1b7c2c8833747ce706e1e0ce/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", size = 2286703, upload-time = "2026-05-06T13:37:53.304Z" }, - { url = "https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", size = 2092042, upload-time = "2026-05-06T13:38:46.981Z" }, - { url = "https://files.pythonhosted.org/packages/a9/da/0a422b57bf8504102bf3c4ccea9c41bab5a5cee6a54650acf8faf67f5a24/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", size = 2117231, upload-time = "2026-05-06T13:39:23.146Z" }, - { url = "https://files.pythonhosted.org/packages/bd/2a/2ac13c3af305843e23c5078c53d135656b3f05a2fd78cb7bbbb12e97b473/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", size = 2168388, upload-time = "2026-05-06T13:40:08.06Z" }, - { url = "https://files.pythonhosted.org/packages/72/04/2beacf7e1607e93eefe4aed1b4709f079b905fb77530179d4f7c71745f22/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", size = 2184769, upload-time = "2026-05-06T13:38:13.901Z" }, - { url = "https://files.pythonhosted.org/packages/9e/29/d2b9fd9f539133548eaf622c06a4ce176cb46ac59f32d0359c4abc0de047/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", size = 2319312, upload-time = "2026-05-06T13:39:08.24Z" }, - { url = "https://files.pythonhosted.org/packages/7c/af/0f7a5b85fec6075bea96e3ef9187de38fccced0de92c1e7feda8d5cc7bb9/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", size = 2361817, upload-time = "2026-05-06T13:38:43.2Z" }, - { url = "https://files.pythonhosted.org/packages/25/a4/73363fec545fd3ec025490bdda2743c56d0dd5b6266b1a53bbe9e4265375/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", size = 1987085, upload-time = "2026-05-06T13:39:25.497Z" }, - { url = "https://files.pythonhosted.org/packages/01/aa/62f082da2c91fac1c234bc9ee0066257ce83f0604abd72e4c9d5991f2d84/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", size = 2074311, upload-time = "2026-05-06T13:39:59.922Z" }, - { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, - { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, - { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, - { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, - { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, - { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, - { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, - { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, - { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, - { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, - { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, - { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, - { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, - { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, - { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, - { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, - { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, - { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, - { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, - { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, - { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, - { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, - { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, - { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, - { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, - { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, - { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, - { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, - { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, - { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, - { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, - { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, - { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, - { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, - { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, - { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, - { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, - { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, - { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, - { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, - { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, - { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, - { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, - { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, - { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, - { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, - { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, - { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, - { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, - { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, - { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, - { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, - { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, - { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, - { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, - { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, - { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, - { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, - { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, - { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, - { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, - { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, - { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, - { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, - { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, - { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, - { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, - { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, - { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, - { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, - { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, - { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, - { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, - { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, - { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, - { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, - { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, - { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, - { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, - { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, - { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, - { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, - { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, - { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, - { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, - { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, - { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, - { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, + { url = "https://files.pythonhosted.org/packages/22/98/b50eb9a411e87483b5c65dba4fa430a06bac4234d3403a40e5a9905ebcd0/pydantic_core-2.46.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:1da3786b8018e60349680720158cc19161cc3b4bdd815beb0a321cd5ce1ad5b1", size = 2108971, upload-time = "2026-04-20T14:43:51.945Z" }, + { url = "https://files.pythonhosted.org/packages/08/4b/f364b9d161718ff2217160a4b5d41ce38de60aed91c3689ebffa1c939d23/pydantic_core-2.46.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cc0988cb29d21bf4a9d5cf2ef970b5c0e38d8d8e107a493278c05dc6c1dda69f", size = 1949588, upload-time = "2026-04-20T14:44:10.386Z" }, + { url = "https://files.pythonhosted.org/packages/8f/8b/30bd03ee83b2f5e29f5ba8e647ab3c456bf56f2ec72fdbcc0215484a0854/pydantic_core-2.46.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27f9067c3bfadd04c55484b89c0d267981b2f3512850f6f66e1e74204a4e4ce3", size = 1975986, upload-time = "2026-04-20T14:43:57.106Z" }, + { url = "https://files.pythonhosted.org/packages/3c/54/13ccf954d84ec275d5d023d5786e4aa48840bc9f161f2838dc98e1153518/pydantic_core-2.46.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a642ac886ecf6402d9882d10c405dcf4b902abeb2972cd5fb4a48c83cd59279a", size = 2055830, upload-time = "2026-04-20T14:44:15.499Z" }, + { url = "https://files.pythonhosted.org/packages/be/0e/65f38125e660fdbd72aa858e7dfae893645cfa0e7b13d333e174a367cd23/pydantic_core-2.46.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79f561438481f28681584b89e2effb22855e2179880314bcddbf5968e935e807", size = 2222340, upload-time = "2026-04-20T14:41:51.353Z" }, + { url = "https://files.pythonhosted.org/packages/d1/88/f3ab7739efe0e7e80777dbb84c59eb98518e3f57ea433206194c2e425272/pydantic_core-2.46.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57a973eae4665352a47cf1a99b4ee864620f2fe663a217d7a8da68a1f3a5bfda", size = 2280727, upload-time = "2026-04-20T14:41:30.461Z" }, + { url = "https://files.pythonhosted.org/packages/2a/6d/c228219080817bec4982f9531cadb18da6aaa770fdeb114f49c237ac2c9f/pydantic_core-2.46.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83d002b97072a53ea150d63e0a3adfae5670cef5aa8a6e490240e482d3b22e57", size = 2092158, upload-time = "2026-04-20T14:44:07.305Z" }, + { url = "https://files.pythonhosted.org/packages/0f/b1/525a16711e7c6d61635fac3b0bd54600b5c5d9f60c6fc5aaab26b64a2297/pydantic_core-2.46.3-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b40ddd51e7c44b28cfaef746c9d3c506d658885e0a46f9eeef2ee815cbf8e045", size = 2116626, upload-time = "2026-04-20T14:42:34.118Z" }, + { url = "https://files.pythonhosted.org/packages/ef/7c/17d30673351439a6951bf54f564cf2443ab00ae264ec9df00e2efd710eb5/pydantic_core-2.46.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ac5ec7fb9b87f04ee839af2d53bcadea57ded7d229719f56c0ed895bff987943", size = 2160691, upload-time = "2026-04-20T14:41:14.023Z" }, + { url = "https://files.pythonhosted.org/packages/86/66/af8adbcbc0886ead7f1a116606a534d75a307e71e6e08226000d51b880d2/pydantic_core-2.46.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a3b11c812f61b3129c4905781a2601dfdfdea5fe1e6c1cfb696b55d14e9c054f", size = 2182543, upload-time = "2026-04-20T14:40:48.886Z" }, + { url = "https://files.pythonhosted.org/packages/b0/37/6de71e0f54c54a4190010f57deb749e1ddf75c568ada3b1320b70067f121/pydantic_core-2.46.3-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:1108da631e602e5b3c38d6d04fe5bb3bfa54349e6918e3ca6cf570b2e2b2f9d4", size = 2324513, upload-time = "2026-04-20T14:42:36.121Z" }, + { url = "https://files.pythonhosted.org/packages/51/b1/9fc74ce94f603d5ef59ff258ca9c2c8fb902fb548d340a96f77f4d1c3b7f/pydantic_core-2.46.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:de885175515bcfa98ae618c1df7a072f13d179f81376c8007112af20567fd08a", size = 2361853, upload-time = "2026-04-20T14:43:24.886Z" }, + { url = "https://files.pythonhosted.org/packages/40/d0/4c652fc592db35f100279ee751d5a145aca1b9a7984b9684ba7c1b5b0535/pydantic_core-2.46.3-cp310-cp310-win32.whl", hash = "sha256:d11058e3201527d41bc6b545c79187c9e4bf85e15a236a6007f0e991518882b7", size = 1980465, upload-time = "2026-04-20T14:44:46.239Z" }, + { url = "https://files.pythonhosted.org/packages/27/b8/a920453c38afbe1f355e1ea0b0d94a0a3e0b0879d32d793108755fa171d5/pydantic_core-2.46.3-cp310-cp310-win_amd64.whl", hash = "sha256:3612edf65c8ea67ac13616c4d23af12faef1ae435a8a93e5934c2a0cbbdd1fd6", size = 2073884, upload-time = "2026-04-20T14:43:01.201Z" }, + { url = "https://files.pythonhosted.org/packages/22/a2/1ba90a83e85a3f94c796b184f3efde9c72f2830dcda493eea8d59ba78e6d/pydantic_core-2.46.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ab124d49d0459b2373ecf54118a45c28a1e6d4192a533fbc915e70f556feb8e5", size = 2106740, upload-time = "2026-04-20T14:41:20.932Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f6/99ae893c89a0b9d3daec9f95487aa676709aa83f67643b3f0abaf4ab628a/pydantic_core-2.46.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cca67d52a5c7a16aed2b3999e719c4bcf644074eac304a5d3d62dd70ae7d4b2c", size = 1948293, upload-time = "2026-04-20T14:43:42.115Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b8/2e8e636dc9e3f16c2e16bf0849e24be82c5ee82c603c65fc0326666328fc/pydantic_core-2.46.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c024e08c0ba23e6fd68c771a521e9d6a792f2ebb0fa734296b36394dc30390e", size = 1973222, upload-time = "2026-04-20T14:41:57.841Z" }, + { url = "https://files.pythonhosted.org/packages/34/36/0e730beec4d83c5306f417afbd82ff237d9a21e83c5edf675f31ed84c1fe/pydantic_core-2.46.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6645ce7eec4928e29a1e3b3d5c946621d105d3e79f0c9cddf07c2a9770949287", size = 2053852, upload-time = "2026-04-20T14:40:43.077Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f0/3071131f47e39136a17814576e0fada9168569f7f8c0e6ac4d1ede6a4958/pydantic_core-2.46.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a712c7118e6c5ea96562f7b488435172abb94a3c53c22c9efc1412264a45cbbe", size = 2221134, upload-time = "2026-04-20T14:43:03.349Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a9/a2dc023eec5aa4b02a467874bad32e2446957d2adcab14e107eab502e978/pydantic_core-2.46.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:69a868ef3ff206343579021c40faf3b1edc64b1cc508ff243a28b0a514ccb050", size = 2279785, upload-time = "2026-04-20T14:41:19.285Z" }, + { url = "https://files.pythonhosted.org/packages/0a/44/93f489d16fb63fbd41c670441536541f6e8cfa1e5a69f40bc9c5d30d8c90/pydantic_core-2.46.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc7e8c32db809aa0f6ea1d6869ebc8518a65d5150fdfad8bcae6a49ae32a22e2", size = 2089404, upload-time = "2026-04-20T14:43:10.108Z" }, + { url = "https://files.pythonhosted.org/packages/2a/78/8692e3aa72b2d004f7a5d937f1dfdc8552ba26caf0bec75f342c40f00dec/pydantic_core-2.46.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:3481bd1341dc85779ee506bc8e1196a277ace359d89d28588a9468c3ecbe63fa", size = 2114898, upload-time = "2026-04-20T14:44:51.475Z" }, + { url = "https://files.pythonhosted.org/packages/6a/62/e83133f2e7832532060175cebf1f13748f4c7e7e7165cdd1f611f174494b/pydantic_core-2.46.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8690eba565c6d68ffd3a8655525cbdd5246510b44a637ee2c6c03a7ebfe64d3c", size = 2157856, upload-time = "2026-04-20T14:43:46.64Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ec/6a500e3ad7718ee50583fae79c8651f5d37e3abce1fa9ae177ae65842c53/pydantic_core-2.46.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4de88889d7e88d50d40ee5b39d5dac0bcaef9ba91f7e536ac064e6b2834ecccf", size = 2180168, upload-time = "2026-04-20T14:42:00.302Z" }, + { url = "https://files.pythonhosted.org/packages/d8/53/8267811054b1aa7fc1dc7ded93812372ef79a839f5e23558136a6afbfde1/pydantic_core-2.46.3-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:e480080975c1ef7f780b8f99ed72337e7cc5efea2e518a20a692e8e7b278eb8b", size = 2322885, upload-time = "2026-04-20T14:41:05.253Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c1/1c0acdb3aa0856ddc4ecc55214578f896f2de16f400cf51627eb3c26c1c4/pydantic_core-2.46.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:de3a5c376f8cd94da9a1b8fd3dd1c16c7a7b216ed31dc8ce9fd7a22bf13b836e", size = 2360328, upload-time = "2026-04-20T14:41:43.991Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d0/ef39cd0f4a926814f360e71c1adeab48ad214d9727e4deb48eedfb5bce1a/pydantic_core-2.46.3-cp311-cp311-win32.whl", hash = "sha256:fc331a5314ffddd5385b9ee9d0d2fee0b13c27e0e02dad71b1ae5d6561f51eeb", size = 1979464, upload-time = "2026-04-20T14:43:12.215Z" }, + { url = "https://files.pythonhosted.org/packages/18/9c/f41951b0d858e343f1cf09398b2a7b3014013799744f2c4a8ad6a3eec4f2/pydantic_core-2.46.3-cp311-cp311-win_amd64.whl", hash = "sha256:b5b9c6cf08a8a5e502698f5e153056d12c34b8fb30317e0c5fd06f45162a6346", size = 2070837, upload-time = "2026-04-20T14:41:47.707Z" }, + { url = "https://files.pythonhosted.org/packages/9f/1e/264a17cd582f6ed50950d4d03dd5fefd84e570e238afe1cb3e25cf238769/pydantic_core-2.46.3-cp311-cp311-win_arm64.whl", hash = "sha256:5dfd51cf457482f04ec49491811a2b8fd5b843b64b11eecd2d7a1ee596ea78a6", size = 2053647, upload-time = "2026-04-20T14:42:27.535Z" }, + { url = "https://files.pythonhosted.org/packages/4b/cb/5b47425556ecc1f3fe18ed2a0083188aa46e1dd812b06e406475b3a5d536/pydantic_core-2.46.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b11b59b3eee90a80a36701ddb4576d9ae31f93f05cb9e277ceaa09e6bf074a67", size = 2101946, upload-time = "2026-04-20T14:40:52.581Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4f/2fb62c2267cae99b815bbf4a7b9283812c88ca3153ef29f7707200f1d4e5/pydantic_core-2.46.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:af8653713055ea18a3abc1537fe2ebc42f5b0bbb768d1eb79fd74eb47c0ac089", size = 1951612, upload-time = "2026-04-20T14:42:42.996Z" }, + { url = "https://files.pythonhosted.org/packages/50/6e/b7348fd30d6556d132cddd5bd79f37f96f2601fe0608afac4f5fb01ec0b3/pydantic_core-2.46.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75a519dab6d63c514f3a81053e5266c549679e4aa88f6ec57f2b7b854aceb1b0", size = 1977027, upload-time = "2026-04-20T14:42:02.001Z" }, + { url = "https://files.pythonhosted.org/packages/82/11/31d60ee2b45540d3fb0b29302a393dbc01cd771c473f5b5147bcd353e593/pydantic_core-2.46.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a6cd87cb1575b1ad05ba98894c5b5c96411ef678fa2f6ed2576607095b8d9789", size = 2063008, upload-time = "2026-04-20T14:44:17.952Z" }, + { url = "https://files.pythonhosted.org/packages/8a/db/3a9d1957181b59258f44a2300ab0f0be9d1e12d662a4f57bb31250455c52/pydantic_core-2.46.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f80a55484b8d843c8ada81ebf70a682f3f00a3d40e378c06cf17ecb44d280d7d", size = 2233082, upload-time = "2026-04-20T14:40:57.934Z" }, + { url = "https://files.pythonhosted.org/packages/9c/e1/3277c38792aeb5cfb18c2f0c5785a221d9ff4e149abbe1184d53d5f72273/pydantic_core-2.46.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3861f1731b90c50a3266316b9044f5c9b405eecb8e299b0a7120596334e4fe9c", size = 2304615, upload-time = "2026-04-20T14:42:12.584Z" }, + { url = "https://files.pythonhosted.org/packages/5e/d5/e3d9717c9eba10855325650afd2a9cba8e607321697f18953af9d562da2f/pydantic_core-2.46.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb528e295ed31570ac3dcc9bfdd6e0150bc11ce6168ac87a8082055cf1a67395", size = 2094380, upload-time = "2026-04-20T14:43:05.522Z" }, + { url = "https://files.pythonhosted.org/packages/a1/20/abac35dedcbfd66c6f0b03e4e3564511771d6c9b7ede10a362d03e110d9b/pydantic_core-2.46.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:367508faa4973b992b271ba1494acaab36eb7e8739d1e47be5035fb1ea225396", size = 2135429, upload-time = "2026-04-20T14:41:55.549Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a5/41bfd1df69afad71b5cf0535055bccc73022715ad362edbc124bc1e021d7/pydantic_core-2.46.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ad3c826fe523e4becf4fe39baa44286cff85ef137c729a2c5e269afbfd0905d", size = 2174582, upload-time = "2026-04-20T14:41:45.96Z" }, + { url = "https://files.pythonhosted.org/packages/79/65/38d86ea056b29b2b10734eb23329b7a7672ca604df4f2b6e9c02d4ee22fe/pydantic_core-2.46.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ec638c5d194ef8af27db69f16c954a09797c0dc25015ad6123eb2c73a4d271ca", size = 2187533, upload-time = "2026-04-20T14:40:55.367Z" }, + { url = "https://files.pythonhosted.org/packages/b6/55/a1129141678a2026badc539ad1dee0a71d06f54c2f06a4bd68c030ac781b/pydantic_core-2.46.3-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:28ed528c45446062ee66edb1d33df5d88828ae167de76e773a3c7f64bd14e976", size = 2332985, upload-time = "2026-04-20T14:44:13.05Z" }, + { url = "https://files.pythonhosted.org/packages/d7/60/cb26f4077719f709e54819f4e8e1d43f4091f94e285eb6bd21e1190a7b7c/pydantic_core-2.46.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aed19d0c783886d5bd86d80ae5030006b45e28464218747dcf83dabfdd092c7b", size = 2373670, upload-time = "2026-04-20T14:41:53.421Z" }, + { url = "https://files.pythonhosted.org/packages/6b/7e/c3f21882bdf1d8d086876f81b5e296206c69c6082551d776895de7801fa0/pydantic_core-2.46.3-cp312-cp312-win32.whl", hash = "sha256:06d5d8820cbbdb4147578c1fe7ffcd5b83f34508cb9f9ab76e807be7db6ff0a4", size = 1966722, upload-time = "2026-04-20T14:44:30.588Z" }, + { url = "https://files.pythonhosted.org/packages/57/be/6b5e757b859013ebfbd7adba02f23b428f37c86dcbf78b5bb0b4ffd36e99/pydantic_core-2.46.3-cp312-cp312-win_amd64.whl", hash = "sha256:c3212fda0ee959c1dd04c60b601ec31097aaa893573a3a1abd0a47bcac2968c1", size = 2072970, upload-time = "2026-04-20T14:42:54.248Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f8/a989b21cc75e9a32d24192ef700eea606521221a89faa40c919ce884f2b1/pydantic_core-2.46.3-cp312-cp312-win_arm64.whl", hash = "sha256:f1f8338dd7a7f31761f1f1a3c47503a9a3b34eea3c8b01fa6ee96408affb5e72", size = 2035963, upload-time = "2026-04-20T14:44:20.4Z" }, + { url = "https://files.pythonhosted.org/packages/9b/3c/9b5e8eb9821936d065439c3b0fb1490ffa64163bfe7e1595985a47896073/pydantic_core-2.46.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:12bc98de041458b80c86c56b24df1d23832f3e166cbaff011f25d187f5c62c37", size = 2102109, upload-time = "2026-04-20T14:41:24.219Z" }, + { url = "https://files.pythonhosted.org/packages/91/97/1c41d1f5a19f241d8069f1e249853bcce378cdb76eec8ab636d7bc426280/pydantic_core-2.46.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:85348b8f89d2c3508b65b16c3c33a4da22b8215138d8b996912bb1532868885f", size = 1951820, upload-time = "2026-04-20T14:42:14.236Z" }, + { url = "https://files.pythonhosted.org/packages/30/b4/d03a7ae14571bc2b6b3c7b122441154720619afe9a336fa3a95434df5e2f/pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1105677a6df914b1fb71a81b96c8cce7726857e1717d86001f29be06a25ee6f8", size = 1977785, upload-time = "2026-04-20T14:42:31.648Z" }, + { url = "https://files.pythonhosted.org/packages/ae/0c/4086f808834b59e3c8f1aa26df8f4b6d998cdcf354a143d18ef41529d1fe/pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:87082cd65669a33adeba5470769e9704c7cf026cc30afb9cc77fd865578ebaad", size = 2062761, upload-time = "2026-04-20T14:40:37.093Z" }, + { url = "https://files.pythonhosted.org/packages/fa/71/a649be5a5064c2df0db06e0a512c2281134ed2fcc981f52a657936a7527c/pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60e5f66e12c4f5212d08522963380eaaeac5ebd795826cfd19b2dfb0c7a52b9c", size = 2232989, upload-time = "2026-04-20T14:42:59.254Z" }, + { url = "https://files.pythonhosted.org/packages/a2/84/7756e75763e810b3a710f4724441d1ecc5883b94aacb07ca71c5fb5cfb69/pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b6cdf19bf84128d5e7c37e8a73a0c5c10d51103a650ac585d42dd6ae233f2b7f", size = 2303975, upload-time = "2026-04-20T14:41:32.287Z" }, + { url = "https://files.pythonhosted.org/packages/6c/35/68a762e0c1e31f35fa0dac733cbd9f5b118042853698de9509c8e5bf128b/pydantic_core-2.46.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:031bb17f4885a43773c8c763089499f242aee2ea85cf17154168775dccdecf35", size = 2095325, upload-time = "2026-04-20T14:42:47.685Z" }, + { url = "https://files.pythonhosted.org/packages/77/bf/1bf8c9a8e91836c926eae5e3e51dce009bf495a60ca56060689d3df3f340/pydantic_core-2.46.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:bcf2a8b2982a6673693eae7348ef3d8cf3979c1d63b54fca7c397a635cc68687", size = 2133368, upload-time = "2026-04-20T14:41:22.766Z" }, + { url = "https://files.pythonhosted.org/packages/e5/50/87d818d6bab915984995157ceb2380f5aac4e563dddbed6b56f0ed057aba/pydantic_core-2.46.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28e8cf2f52d72ced402a137145923a762cbb5081e48b34312f7a0c8f55928ec3", size = 2173908, upload-time = "2026-04-20T14:42:52.044Z" }, + { url = "https://files.pythonhosted.org/packages/91/88/a311fb306d0bd6185db41fa14ae888fb81d0baf648a761ae760d30819d33/pydantic_core-2.46.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:17eaface65d9fc5abb940003020309c1bf7a211f5f608d7870297c367e6f9022", size = 2186422, upload-time = "2026-04-20T14:43:29.55Z" }, + { url = "https://files.pythonhosted.org/packages/8f/79/28fd0d81508525ab2054fef7c77a638c8b5b0afcbbaeee493cf7c3fef7e1/pydantic_core-2.46.3-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:93fd339f23408a07e98950a89644f92c54d8729719a40b30c0a30bb9ebc55d23", size = 2332709, upload-time = "2026-04-20T14:42:16.134Z" }, + { url = "https://files.pythonhosted.org/packages/b3/21/795bf5fe5c0f379308b8ef19c50dedab2e7711dbc8d0c2acf08f1c7daa05/pydantic_core-2.46.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:23cbdb3aaa74dfe0837975dbf69b469753bbde8eacace524519ffdb6b6e89eb7", size = 2372428, upload-time = "2026-04-20T14:41:10.974Z" }, + { url = "https://files.pythonhosted.org/packages/45/b3/ed14c659cbe7605e3ef063077680a64680aec81eb1a04763a05190d49b7f/pydantic_core-2.46.3-cp313-cp313-win32.whl", hash = "sha256:610eda2e3838f401105e6326ca304f5da1e15393ae25dacae5c5c63f2c275b13", size = 1965601, upload-time = "2026-04-20T14:41:42.128Z" }, + { url = "https://files.pythonhosted.org/packages/ef/bb/adb70d9a762ddd002d723fbf1bd492244d37da41e3af7b74ad212609027e/pydantic_core-2.46.3-cp313-cp313-win_amd64.whl", hash = "sha256:68cc7866ed863db34351294187f9b729964c371ba33e31c26f478471c52e1ed0", size = 2071517, upload-time = "2026-04-20T14:43:36.096Z" }, + { url = "https://files.pythonhosted.org/packages/52/eb/66faefabebfe68bd7788339c9c9127231e680b11906368c67ce112fdb47f/pydantic_core-2.46.3-cp313-cp313-win_arm64.whl", hash = "sha256:f64b5537ac62b231572879cd08ec05600308636a5d63bcbdb15063a466977bec", size = 2035802, upload-time = "2026-04-20T14:43:38.507Z" }, + { url = "https://files.pythonhosted.org/packages/7f/db/a7bcb4940183fda36022cd18ba8dd12f2dff40740ec7b58ce7457befa416/pydantic_core-2.46.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:afa3aa644f74e290cdede48a7b0bee37d1c35e71b05105f6b340d484af536d9b", size = 2097614, upload-time = "2026-04-20T14:44:38.374Z" }, + { url = "https://files.pythonhosted.org/packages/24/35/e4066358a22e3e99519db370494c7528f5a2aa1367370e80e27e20283543/pydantic_core-2.46.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ced3310e51aa425f7f77da8bbbb5212616655bedbe82c70944320bc1dbe5e018", size = 1951896, upload-time = "2026-04-20T14:40:53.996Z" }, + { url = "https://files.pythonhosted.org/packages/87/92/37cf4049d1636996e4b888c05a501f40a43ff218983a551d57f9d5e14f0d/pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e29908922ce9da1a30b4da490bd1d3d82c01dcfdf864d2a74aacee674d0bfa34", size = 1979314, upload-time = "2026-04-20T14:41:49.446Z" }, + { url = "https://files.pythonhosted.org/packages/d8/36/9ff4d676dfbdfb2d591cf43f3d90ded01e15b1404fd101180ed2d62a2fd3/pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0c9ff69140423eea8ed2d5477df3ba037f671f5e897d206d921bc9fdc39613e7", size = 2056133, upload-time = "2026-04-20T14:42:23.574Z" }, + { url = "https://files.pythonhosted.org/packages/bc/f0/405b442a4d7ba855b06eec8b2bf9c617d43b8432d099dfdc7bf999293495/pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b675ab0a0d5b1c8fdb81195dc5bcefea3f3c240871cdd7ff9a2de8aa50772eb2", size = 2228726, upload-time = "2026-04-20T14:44:22.816Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f8/65cd92dd5a0bd89ba277a98ecbfaf6fc36bbd3300973c7a4b826d6ab1391/pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0087084960f209a9a4af50ecd1fb063d9ad3658c07bb81a7a53f452dacbfb2ba", size = 2301214, upload-time = "2026-04-20T14:44:48.792Z" }, + { url = "https://files.pythonhosted.org/packages/fd/86/ef96a4c6e79e7a2d0410826a68fbc0eccc0fd44aa733be199d5fcac3bb87/pydantic_core-2.46.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed42e6cc8e1b0e2b9b96e2276bad70ae625d10d6d524aed0c93de974ae029f9f", size = 2099927, upload-time = "2026-04-20T14:41:40.196Z" }, + { url = "https://files.pythonhosted.org/packages/6d/53/269caf30e0096e0a8a8f929d1982a27b3879872cca2d917d17c2f9fdf4fe/pydantic_core-2.46.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:f1771ce258afb3e4201e67d154edbbae712a76a6081079fe247c2f53c6322c22", size = 2128789, upload-time = "2026-04-20T14:41:15.868Z" }, + { url = "https://files.pythonhosted.org/packages/00/b0/1a6d9b6a587e118482910c244a1c5acf4d192604174132efd12bf0ac486f/pydantic_core-2.46.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a7610b6a5242a6c736d8ad47fd5fff87fcfe8f833b281b1c409c3d6835d9227f", size = 2173815, upload-time = "2026-04-20T14:44:25.152Z" }, + { url = "https://files.pythonhosted.org/packages/87/56/e7e00d4041a7e62b5a40815590114db3b535bf3ca0bf4dca9f16cef25246/pydantic_core-2.46.3-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:ff5e7783bcc5476e1db448bf268f11cb257b1c276d3e89f00b5727be86dd0127", size = 2181608, upload-time = "2026-04-20T14:41:28.933Z" }, + { url = "https://files.pythonhosted.org/packages/e8/22/4bd23c3d41f7c185d60808a1de83c76cf5aeabf792f6c636a55c3b1ec7f9/pydantic_core-2.46.3-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:9d2e32edcc143bc01e95300671915d9ca052d4f745aa0a49c48d4803f8a85f2c", size = 2326968, upload-time = "2026-04-20T14:42:03.962Z" }, + { url = "https://files.pythonhosted.org/packages/24/ac/66cd45129e3915e5ade3b292cb3bc7fd537f58f8f8dbdaba6170f7cabb74/pydantic_core-2.46.3-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:6e42d83d1c6b87fa56b521479cff237e626a292f3b31b6345c15a99121b454c1", size = 2369842, upload-time = "2026-04-20T14:41:35.52Z" }, + { url = "https://files.pythonhosted.org/packages/a2/51/dd4248abb84113615473aa20d5545b7c4cd73c8644003b5259686f93996c/pydantic_core-2.46.3-cp314-cp314-win32.whl", hash = "sha256:07bc6d2a28c3adb4f7c6ae46aa4f2d2929af127f587ed44057af50bf1ce0f505", size = 1959661, upload-time = "2026-04-20T14:41:00.042Z" }, + { url = "https://files.pythonhosted.org/packages/20/eb/59980e5f1ae54a3b86372bd9f0fa373ea2d402e8cdcd3459334430f91e91/pydantic_core-2.46.3-cp314-cp314-win_amd64.whl", hash = "sha256:8940562319bc621da30714617e6a7eaa6b98c84e8c685bcdc02d7ed5e7c7c44e", size = 2071686, upload-time = "2026-04-20T14:43:16.471Z" }, + { url = "https://files.pythonhosted.org/packages/8c/db/1cf77e5247047dfee34bc01fa9bca134854f528c8eb053e144298893d370/pydantic_core-2.46.3-cp314-cp314-win_arm64.whl", hash = "sha256:5dcbbcf4d22210ced8f837c96db941bdb078f419543472aca5d9a0bb7cddc7df", size = 2026907, upload-time = "2026-04-20T14:43:31.732Z" }, + { url = "https://files.pythonhosted.org/packages/57/c0/b3df9f6a543276eadba0a48487b082ca1f201745329d97dbfa287034a230/pydantic_core-2.46.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:d0fe3dce1e836e418f912c1ad91c73357d03e556a4d286f441bf34fed2dbeecf", size = 2095047, upload-time = "2026-04-20T14:42:37.982Z" }, + { url = "https://files.pythonhosted.org/packages/66/57/886a938073b97556c168fd99e1a7305bb363cd30a6d2c76086bf0587b32a/pydantic_core-2.46.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9ce92e58abc722dac1bf835a6798a60b294e48eb0e625ec9fd994b932ac5feee", size = 1934329, upload-time = "2026-04-20T14:43:49.655Z" }, + { url = "https://files.pythonhosted.org/packages/0b/7c/b42eaa5c34b13b07ecb51da21761297a9b8eb43044c864a035999998f328/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a03e6467f0f5ab796a486146d1b887b2dc5e5f9b3288898c1b1c3ad974e53e4a", size = 1974847, upload-time = "2026-04-20T14:42:10.737Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9b/92b42db6543e7de4f99ae977101a2967b63122d4b6cf7773812da2d7d5b5/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2798b6ba041b9d70acfb9071a2ea13c8456dd1e6a5555798e41ba7b0790e329c", size = 2041742, upload-time = "2026-04-20T14:40:44.262Z" }, + { url = "https://files.pythonhosted.org/packages/0f/19/46fbe1efabb5aa2834b43b9454e70f9a83ad9c338c1291e48bdc4fecf167/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9be3e221bdc6d69abf294dcf7aff6af19c31a5cdcc8f0aa3b14be29df4bd03b1", size = 2236235, upload-time = "2026-04-20T14:41:27.307Z" }, + { url = "https://files.pythonhosted.org/packages/77/da/b3f95bc009ad60ec53120f5d16c6faa8cabdbe8a20d83849a1f2b8728148/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f13936129ce841f2a5ddf6f126fea3c43cd128807b5a59588c37cf10178c2e64", size = 2282633, upload-time = "2026-04-20T14:44:33.271Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6e/401336117722e28f32fb8220df676769d28ebdf08f2f4469646d404c43a3/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28b5f2ef03416facccb1c6ef744c69793175fd27e44ef15669201601cf423acb", size = 2109679, upload-time = "2026-04-20T14:44:41.065Z" }, + { url = "https://files.pythonhosted.org/packages/fc/53/b289f9bc8756a32fe718c46f55afaeaf8d489ee18d1a1e7be1db73f42cc4/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:830d1247d77ad23852314f069e9d7ddafeec5f684baf9d7e7065ed46a049c4e6", size = 2108342, upload-time = "2026-04-20T14:42:50.144Z" }, + { url = "https://files.pythonhosted.org/packages/10/5b/8292fc7c1f9111f1b2b7c1b0dcf1179edcd014fc3ea4517499f50b829d71/pydantic_core-2.46.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0793c90c1a3c74966e7975eaef3ed30ebdff3260a0f815a62a22adc17e4c01c", size = 2157208, upload-time = "2026-04-20T14:42:08.133Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/f80044e9ec07580f057a89fc131f78dda7a58751ddf52bbe05eaf31db50f/pydantic_core-2.46.3-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:d2d0aead851b66f5245ec0c4fb2612ef457f8bbafefdf65a2bf9d6bac6140f47", size = 2167237, upload-time = "2026-04-20T14:42:25.412Z" }, + { url = "https://files.pythonhosted.org/packages/f8/84/6781a1b037f3b96be9227edbd1101f6d3946746056231bf4ac48cdff1a8d/pydantic_core-2.46.3-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:2f40e4246676beb31c5ce77c38a55ca4e465c6b38d11ea1bd935420568e0b1ab", size = 2312540, upload-time = "2026-04-20T14:40:40.313Z" }, + { url = "https://files.pythonhosted.org/packages/3e/db/19c0839feeb728e7df03255581f198dfdf1c2aeb1e174a8420b63c5252e5/pydantic_core-2.46.3-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:cf489cf8986c543939aeee17a09c04d6ffb43bfef8ca16fcbcc5cfdcbed24dba", size = 2369556, upload-time = "2026-04-20T14:41:09.427Z" }, + { url = "https://files.pythonhosted.org/packages/e0/15/3228774cb7cd45f5f721ddf1b2242747f4eb834d0c491f0c02d606f09fed/pydantic_core-2.46.3-cp314-cp314t-win32.whl", hash = "sha256:ffe0883b56cfc05798bf994164d2b2ff03efe2d22022a2bb080f3b626176dd56", size = 1949756, upload-time = "2026-04-20T14:41:25.717Z" }, + { url = "https://files.pythonhosted.org/packages/b8/2a/c79cf53fd91e5a87e30d481809f52f9a60dd221e39de66455cf04deaad37/pydantic_core-2.46.3-cp314-cp314t-win_amd64.whl", hash = "sha256:706d9d0ce9cf4593d07270d8e9f53b161f90c57d315aeec4fb4fd7a8b10240d8", size = 2051305, upload-time = "2026-04-20T14:43:18.627Z" }, + { url = "https://files.pythonhosted.org/packages/0b/db/d8182a7f1d9343a032265aae186eb063fe26ca4c40f256b21e8da4498e89/pydantic_core-2.46.3-cp314-cp314t-win_arm64.whl", hash = "sha256:77706aeb41df6a76568434701e0917da10692da28cb69d5fb6919ce5fdb07374", size = 2026310, upload-time = "2026-04-20T14:41:01.778Z" }, + { url = "https://files.pythonhosted.org/packages/66/7f/03dbad45cd3aa9083fbc93c210ae8b005af67e4136a14186950a747c6874/pydantic_core-2.46.3-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:9715525891ed524a0a1eb6d053c74d4d4ad5017677fb00af0b7c2644a31bae46", size = 2105683, upload-time = "2026-04-20T14:42:19.779Z" }, + { url = "https://files.pythonhosted.org/packages/26/22/4dc186ac8ea6b257e9855031f51b62a9637beac4d68ac06bee02f046f836/pydantic_core-2.46.3-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:9d2f400712a99a013aff420ef1eb9be077f8189a36c1e3ef87660b4e1088a874", size = 1940052, upload-time = "2026-04-20T14:43:59.274Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ca/d376391a5aff1f2e8188960d7873543608130a870961c2b6b5236627c116/pydantic_core-2.46.3-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd2aab0e2e9dc2daf36bd2686c982535d5e7b1d930a1344a7bb6e82baab42a76", size = 1988172, upload-time = "2026-04-20T14:41:17.469Z" }, + { url = "https://files.pythonhosted.org/packages/0e/6b/523b9f85c23788755d6ab949329de692a2e3a584bc6beb67fef5e035aa9d/pydantic_core-2.46.3-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e9d76736da5f362fabfeea6a69b13b7f2be405c6d6966f06b2f6bfff7e64531", size = 2128596, upload-time = "2026-04-20T14:40:41.707Z" }, + { url = "https://files.pythonhosted.org/packages/34/42/f426db557e8ab2791bc7562052299944a118655496fbff99914e564c0a94/pydantic_core-2.46.3-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:b12dd51f1187c2eb489af8e20f880362db98e954b54ab792fa5d92e8bcc6b803", size = 2091877, upload-time = "2026-04-20T14:43:27.091Z" }, + { url = "https://files.pythonhosted.org/packages/5c/4f/86a832a9d14df58e663bfdf4627dc00d3317c2bd583c4fb23390b0f04b8e/pydantic_core-2.46.3-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:f00a0961b125f1a47af7bcc17f00782e12f4cd056f83416006b30111d941dfa3", size = 1932428, upload-time = "2026-04-20T14:40:45.781Z" }, + { url = "https://files.pythonhosted.org/packages/11/1a/fe857968954d93fb78e0d4b6df5c988c74c4aaa67181c60be7cfe327c0ca/pydantic_core-2.46.3-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57697d7c056aca4bbb680200f96563e841a6386ac1129370a0102592f4dddff5", size = 1997550, upload-time = "2026-04-20T14:44:02.425Z" }, + { url = "https://files.pythonhosted.org/packages/17/eb/9d89ad2d9b0ba8cd65393d434471621b98912abb10fbe1df08e480ba57b5/pydantic_core-2.46.3-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd35aa21299def8db7ef4fe5c4ff862941a9a158ca7b63d61e66fe67d30416b4", size = 2137657, upload-time = "2026-04-20T14:42:45.149Z" }, + { url = "https://files.pythonhosted.org/packages/1f/da/99d40830684f81dec901cac521b5b91c095394cc1084b9433393cde1c2df/pydantic_core-2.46.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:13afdd885f3d71280cf286b13b310ee0f7ccfefd1dbbb661514a474b726e2f25", size = 2107973, upload-time = "2026-04-20T14:42:06.175Z" }, + { url = "https://files.pythonhosted.org/packages/99/a5/87024121818d75bbb2a98ddbaf638e40e7a18b5e0f5492c9ca4b1b316107/pydantic_core-2.46.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:f91c0aff3e3ee0928edd1232c57f643a7a003e6edf1860bc3afcdc749cb513f3", size = 1947191, upload-time = "2026-04-20T14:43:14.319Z" }, + { url = "https://files.pythonhosted.org/packages/60/62/0c1acfe10945b83a6a59d19fbaa92f48825381509e5701b855c08f13db76/pydantic_core-2.46.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6529d1d128321a58d30afcc97b49e98836542f68dd41b33c2e972bb9e5290536", size = 2123791, upload-time = "2026-04-20T14:43:22.766Z" }, + { url = "https://files.pythonhosted.org/packages/75/3e/3b2393b4c8f44285561dc30b00cf307a56a2eff7c483a824db3b8221ca51/pydantic_core-2.46.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:975c267cff4f7e7272eacbe50f6cc03ca9a3da4c4fbd66fffd89c94c1e311aa1", size = 2153197, upload-time = "2026-04-20T14:44:27.932Z" }, + { url = "https://files.pythonhosted.org/packages/ba/75/5af02fb35505051eee727c061f2881c555ab4f8ddb2d42da715a42c9731b/pydantic_core-2.46.3-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2b8e4f2bbdf71415c544b4b1138b8060db7b6611bc927e8064c769f64bed651c", size = 2181073, upload-time = "2026-04-20T14:43:20.729Z" }, + { url = "https://files.pythonhosted.org/packages/10/92/7e0e1bd9ca3c68305db037560ca2876f89b2647deb2f8b6319005de37505/pydantic_core-2.46.3-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:e61ea8e9fff9606d09178f577ff8ccdd7206ff73d6552bcec18e1033c4254b85", size = 2315886, upload-time = "2026-04-20T14:44:04.826Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d8/101655f27eaf3e44558ead736b2795d12500598beed4683f279396fa186e/pydantic_core-2.46.3-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b504bda01bafc69b6d3c7a0c7f039dcf60f47fab70e06fe23f57b5c75bdc82b8", size = 2360528, upload-time = "2026-04-20T14:40:47.431Z" }, + { url = "https://files.pythonhosted.org/packages/07/0f/1c34a74c8d07136f0d729ffe5e1fdab04fbdaa7684f61a92f92511a84a15/pydantic_core-2.46.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:b00b76f7142fc60c762ce579bd29c8fa44aaa56592dd3c54fab3928d0d4ca6ff", size = 2184144, upload-time = "2026-04-20T14:42:57Z" }, ] [[package]] @@ -1342,9 +1235,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyspark" +version = "4.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py4j" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/bf/58ee13add151469c25825b7125bbf62c3bdcec05eec4d458fcb5c5516066/pyspark-4.1.1.tar.gz", hash = "sha256:77f78984aa84fbe865c717dd37b49913b4e5c97d76ef6824f932f1aefa6621ec", size = 455359625, upload-time = "2026-01-09T09:38:38.28Z" } + [[package]] name = "pytest" -version = "9.1.0" +version = "9.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -1355,9 +1257,9 @@ dependencies = [ { name = "pygments" }, { name = "tomli", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/84/0e/b5858858d74958632c49b72cb25a3976ff9f632397626715be71c89d3971/pytest-9.1.0.tar.gz", hash = "sha256:41dd9148c08072446394cefd3d79701701335a9f4cae69ba92e39f6c7f5c061c", size = 1634181, upload-time = "2026-06-13T18:52:45.983Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/5a/ba30a81239b909821b3153e303e7def45178bf353da4f72380e6c5e8793b/pytest-9.1.0-py3-none-any.whl", hash = "sha256:8ebb0e7888bdf2bdfc602ec51f8f62d50200af37356c74e503c79a94f5c81f32", size = 386453, upload-time = "2026-06-13T18:52:44.045Z" }, + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, ] [[package]] @@ -1374,6 +1276,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "pytest-testmon" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/1d/3e4230cc67cd6205bbe03c3527500c0ccaf7f0c78b436537eac71590ee4a/pytest_testmon-2.2.0.tar.gz", hash = "sha256:01f488e955ed0e0049777bee598bf1f647dd524e06f544c31a24e68f8d775a51", size = 23108, upload-time = "2025-12-01T07:30:24.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/55/ebb3c2f59fb089f08d00f764830d35780fc4e4c41dffcadafa3264682b65/pytest_testmon-2.2.0-py3-none-any.whl", hash = "sha256:2604ca44a54d61a2e830d9ce828b41a837075e4ebc1f81b148add8e90d34815b", size = 25199, upload-time = "2025-12-01T07:30:23.623Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1453,27 +1368,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.15.18" +version = "0.15.12" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/98/1295ad5a5aa9bc85bdcdfa5d82fe7b49c61af5657df4f227637ff9de0da6/ruff-0.15.18.tar.gz", hash = "sha256:2698a964c70e8bf402dcb99c8810472d270d141e7aa8c4e13599fd52033a2f33", size = 4761437, upload-time = "2026-06-18T18:25:39.224Z" } +sdist = { url = "https://files.pythonhosted.org/packages/99/43/3291f1cc9106f4c63bdce7a8d0df5047fe8422a75b091c16b5e9355e0b11/ruff-0.15.12.tar.gz", hash = "sha256:ecea26adb26b4232c0c2ca19ccbc0083a68344180bba2a600605538ce51a40a6", size = 4643852, upload-time = "2026-04-24T18:17:14.305Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/d0/686e984941269621e2be72612d5c1e461f8f7b38415a2a7d7a81c8ae6715/ruff-0.15.18-py3-none-linux_armv6l.whl", hash = "sha256:8b6850172348c8381b8b3084c5915a4393c2373b9b54cd5b5e1ea15812bc10df", size = 10887308, upload-time = "2026-06-18T18:25:03.062Z" }, - { url = "https://files.pythonhosted.org/packages/ed/21/bc4123e3f5515ee99f8ce1eb93a14a0628fe4d1678663cd08f933ac16931/ruff-0.15.18-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3fccc153a85417dcd976883160cacce486997b0a0058dd18f54b8aaaac7d1ce2", size = 11281305, upload-time = "2026-06-18T18:25:30.026Z" }, - { url = "https://files.pythonhosted.org/packages/51/93/4769464c25cf7ab2acb3c7dda9cad3d867eb41c59565b3e2a9d17249c90c/ruff-0.15.18-py3-none-macosx_11_0_arm64.whl", hash = "sha256:08d4c86a68f2c3ec2c9d56380a71fb4a4f65373055cbb8caabd645e9102f38d4", size = 10641215, upload-time = "2026-06-18T18:25:15.802Z" }, - { url = "https://files.pythonhosted.org/packages/6c/42/56926d17120db2c208d76bf60a1a019644dd9e91dc27f0f95c9caddb1366/ruff-0.15.18-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37e5108745c2c0705da916d7d4de533ddf547051ef45f62888c31bae73f66318", size = 10957224, upload-time = "2026-06-18T18:25:36.955Z" }, - { url = "https://files.pythonhosted.org/packages/22/4f/d43fab8d8189afde803103022d000a8ef9f230616d436d52a8b2b8d63b50/ruff-0.15.18-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56949a6ce8b3abde54c0bcb22cebfe57e8771cadc84b407ae8b8eaf67ebdcd43", size = 10699024, upload-time = "2026-06-18T18:25:05.707Z" }, - { url = "https://files.pythonhosted.org/packages/63/42/1e3e4c68bd408b9768cf3e439acbe2c78245225faef253f7028a0cdb63e0/ruff-0.15.18-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01a754cd6a1b630d3f97e33eb452cf7a98040482318e870f8bc52a5a30e62657", size = 11491458, upload-time = "2026-06-18T18:25:20.275Z" }, - { url = "https://files.pythonhosted.org/packages/20/77/47a3484bea8521e14a203d98c389c5c97846675e4f02734672da4a69b52a/ruff-0.15.18-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6ba7a07e03a44dbf10bb086ee06705b173625014ec99f73a7e6836a5e5590a0c", size = 12383752, upload-time = "2026-06-18T18:25:22.535Z" }, - { url = "https://files.pythonhosted.org/packages/0a/ca/054159590787023d83b658a1a1819c4c8910114e7015069340b71c0961cb/ruff-0.15.18-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a2c40a41a4cadbcf5897b548ab29dfe248b20c540961c0247d98a3973c70403", size = 11577923, upload-time = "2026-06-18T18:25:10.702Z" }, - { url = "https://files.pythonhosted.org/packages/6d/ff/d353d6b7bbd73cc0ec37f4463d7540e45e894338abdd9964eee0de332708/ruff-0.15.18-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f0480ce690cbb6c4db6e5d08f19fce98e10ba131a8b60c1bcdac42771e3ae2d", size = 11583925, upload-time = "2026-06-18T18:25:32.391Z" }, - { url = "https://files.pythonhosted.org/packages/c1/4a/891f89b9c296ed3e5f3ece1a5629badc989d9a8fdaa30431aaf4774bc1c2/ruff-0.15.18-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2330215f1f393fa8733f55edce04fcf94c36a2c460fcde31f78cc84e4951e9b1", size = 11582834, upload-time = "2026-06-18T18:25:27.309Z" }, - { url = "https://files.pythonhosted.org/packages/32/a3/ed9e370154bf85de360b93c03026157f02d4943b2d01ff4945f4429f8e8a/ruff-0.15.18-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a6aa6a3d979e48ae617578183674bf264fbe7d0114a796a26bd678d67963c7ff", size = 10927328, upload-time = "2026-06-18T18:25:34.676Z" }, - { url = "https://files.pythonhosted.org/packages/f5/d1/5cf5909329fedb5d39d555ee818ba5cf4638e1a301b89785d34f2905bfcb/ruff-0.15.18-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a81beadbbff2c9c245561ae3f77b16709d87f35eec650d0501679239d3449b22", size = 10693187, upload-time = "2026-06-18T18:25:08.245Z" }, - { url = "https://files.pythonhosted.org/packages/fd/44/ff6c635cf2c4f4e7b618b6640da057376baa36014695487d88aed4794268/ruff-0.15.18-py3-none-musllinux_1_2_i686.whl", hash = "sha256:2186d9e940ae332ab293623a75b5f4fe49565f449954d50a72a046683aa6b809", size = 11208721, upload-time = "2026-06-18T18:25:41.327Z" }, - { url = "https://files.pythonhosted.org/packages/88/d9/5baa2a30861adfb7022cf33c1e35b2fc18085b08c16f83eff4c7b99a5f48/ruff-0.15.18-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5c2abf140438032bc77b2284a6c9944ecd8a19e5f1c7b52b1b8e4a0a80d19a7a", size = 11678599, upload-time = "2026-06-18T18:25:13.607Z" }, - { url = "https://files.pythonhosted.org/packages/c3/1a/0725a7cfdc32ff769efb96ee782bec882e16448c5d9e3be947ec4c04ce27/ruff-0.15.18-py3-none-win32.whl", hash = "sha256:02299e6e9fa5b297a3f6d5d10d7bcd655c925b028bb8b9d4588214549c6b9ec4", size = 10901903, upload-time = "2026-06-18T18:25:24.755Z" }, - { url = "https://files.pythonhosted.org/packages/f3/51/805d9f6fb7970505c3504794a5ec350f605361b807fef4dcf214ebd35e72/ruff-0.15.18-py3-none-win_amd64.whl", hash = "sha256:dac80dc8d26b2257dbefabed62f5d255c3937b4ccb122da1fc634794fa3578b3", size = 12041189, upload-time = "2026-06-18T18:25:17.915Z" }, - { url = "https://files.pythonhosted.org/packages/29/4c/67bb45e41609eb4726f1bfeb59e083cf91d14c696d4bd14c234a980be93d/ruff-0.15.18-py3-none-win_arm64.whl", hash = "sha256:b2c9257fcbd4a3e5b977a1904e6facca016bafe2edc17df24db67cfaee03b4e4", size = 11329958, upload-time = "2026-06-18T18:25:43.686Z" }, + { url = "https://files.pythonhosted.org/packages/c3/6e/e78ffb61d4686f3d96ba3df2c801161843746dcbcbb17a1e927d4829312b/ruff-0.15.12-py3-none-linux_armv6l.whl", hash = "sha256:f86f176e188e94d6bdbc09f09bfd9dc729059ad93d0e7390b5a73efe19f8861c", size = 10640713, upload-time = "2026-04-24T18:17:22.841Z" }, + { url = "https://files.pythonhosted.org/packages/ae/08/a317bc231fb9e7b93e4ef3089501e51922ff88d6936ce5cf870c4fe55419/ruff-0.15.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e3bcd123364c3770b8e1b7baaf343cc99a35f197c5c6e8af79015c666c423a6c", size = 11069267, upload-time = "2026-04-24T18:17:30.105Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/f828e9718d3dce1f5f11c39c4f65afd32783c8b2aebb2e3d259e492c47bd/ruff-0.15.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fe87510d000220aa1ed530d4448a7c696a0cae1213e5ec30e5874287b66557b5", size = 10397182, upload-time = "2026-04-24T18:17:07.177Z" }, + { url = "https://files.pythonhosted.org/packages/71/e0/3310fc6d1b5e1fdea22bf3b1b807c7e187b581021b0d7d4514cccdb5fb71/ruff-0.15.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84a1630093121375a3e2a95b4a6dc7b59e2b4ee76216e32d81aae550a832d002", size = 10758012, upload-time = "2026-04-24T18:16:55.759Z" }, + { url = "https://files.pythonhosted.org/packages/11/c1/a606911aee04c324ddaa883ae418f3569792fd3c4a10c50e0dd0a2311e1e/ruff-0.15.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fb129f40f114f089ebe0ca56c0d251cf2061b17651d464bb6478dc01e69f11f5", size = 10447479, upload-time = "2026-04-24T18:16:51.677Z" }, + { url = "https://files.pythonhosted.org/packages/9d/68/4201e8444f0894f21ab4aeeaee68aa4f10b51613514a20d80bd628d57e88/ruff-0.15.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0c862b172d695db7598426b8af465e7e9ac00a3ea2a3630ee67eb82e366aaa6", size = 11234040, upload-time = "2026-04-24T18:17:16.529Z" }, + { url = "https://files.pythonhosted.org/packages/34/ff/8a6d6cf4ccc23fd67060874e832c18919d1557a0611ebef03fdb01fff11e/ruff-0.15.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2849ea9f3484c3aca43a82f484210370319e7170df4dfe4843395ddf6c57bc33", size = 12087377, upload-time = "2026-04-24T18:17:04.944Z" }, + { url = "https://files.pythonhosted.org/packages/85/f6/c669cf73f5152f623d34e69866a46d5e6185816b19fcd5b6dd8a2d299922/ruff-0.15.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e77c7e51c07fe396826d5969a5b846d9cd4c402535835fb6e21ce8b28fef847", size = 11367784, upload-time = "2026-04-24T18:17:25.409Z" }, + { url = "https://files.pythonhosted.org/packages/e8/39/c61d193b8a1daaa8977f7dea9e8d8ba866e02ea7b65d32f6861693aa4c12/ruff-0.15.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83b2f4f2f3b1026b5fb449b467d9264bf22067b600f7b6f41fc5958909f449d0", size = 11344088, upload-time = "2026-04-24T18:17:12.258Z" }, + { url = "https://files.pythonhosted.org/packages/c2/8d/49afab3645e31e12c590acb6d3b5b69d7aab5b81926dbaf7461f9441f37a/ruff-0.15.12-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9ba3b8f1afd7e2e43d8943e55f249e13f9682fde09711644a6e7290eb4f3e339", size = 11271770, upload-time = "2026-04-24T18:17:02.457Z" }, + { url = "https://files.pythonhosted.org/packages/46/06/33f41fe94403e2b755481cdfb9b7ef3e4e0ed031c4581124658d935d52b4/ruff-0.15.12-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e852ba9fdc890655e1d78f2df1499efbe0e54126bd405362154a75e2bde159c5", size = 10719355, upload-time = "2026-04-24T18:17:27.648Z" }, + { url = "https://files.pythonhosted.org/packages/0d/59/18aa4e014debbf559670e4048e39260a85c7fcee84acfd761ac01e7b8d35/ruff-0.15.12-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dd8aed930da53780d22fc70bdf84452c843cf64f8cb4eb38984319c24c5cd5fd", size = 10462758, upload-time = "2026-04-24T18:17:32.347Z" }, + { url = "https://files.pythonhosted.org/packages/25/e7/cc9f16fd0f3b5fddcbd7ec3d6ae30c8f3fde1047f32a4093a98d633c6570/ruff-0.15.12-py3-none-musllinux_1_2_i686.whl", hash = "sha256:01da3988d225628b709493d7dc67c3b9b12c0210016b08690ef9bd27970b262b", size = 10953498, upload-time = "2026-04-24T18:17:20.674Z" }, + { url = "https://files.pythonhosted.org/packages/72/7a/a9ba7f98c7a575978698f4230c5e8cc54bbc761af34f560818f933dafa0c/ruff-0.15.12-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:9cae0f92bd5700d1213188b31cd3bdd2b315361296d10b96b8e2337d3d11f53e", size = 11447765, upload-time = "2026-04-24T18:17:09.755Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f9/0ae446942c846b8266059ad8a30702a35afae55f5cdc54c5adf8d7afdc27/ruff-0.15.12-py3-none-win32.whl", hash = "sha256:d0185894e038d7043ba8fd6aee7499ece6462dc0ea9f1e260c7451807c714c20", size = 10657277, upload-time = "2026-04-24T18:17:18.591Z" }, + { url = "https://files.pythonhosted.org/packages/33/f1/9614e03e1cdcbf9437570b5400ced8a720b5db22b28d8e0f1bda429f660d/ruff-0.15.12-py3-none-win_amd64.whl", hash = "sha256:c87a162d61ab3adca47c03f7f717c68672edec7d1b5499e652331780fe74950d", size = 11837758, upload-time = "2026-04-24T18:17:00.113Z" }, + { url = "https://files.pythonhosted.org/packages/c0/98/6beb4b351e472e5f4c4613f7c35a5290b8be2497e183825310c4c3a3984b/ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f", size = 11120821, upload-time = "2026-04-24T18:16:57.979Z" }, ] [[package]] @@ -1482,7 +1397,7 @@ version = "2.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/bc/0989043118a27cccb4e906a46b7565ce36ca7b57f5a18b78f4f1b0f72d9d/shapely-2.1.2.tar.gz", hash = "sha256:2ed4ecb28320a433db18a5bf029986aa8afcfd740745e78847e330d5d94922a9", size = 315489, upload-time = "2025-09-24T13:51:41.432Z" } wheels = [ @@ -1546,11 +1461,11 @@ wheels = [ [[package]] name = "snowballstemmer" -version = "3.1.1" +version = "3.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/f8/0a71edf031f03c40db17503cb8ca78a69a171254e568e7db241b0ab57ea1/snowballstemmer-3.1.1.tar.gz", hash = "sha256:e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260", size = 123314, upload-time = "2026-06-03T00:56:40.194Z" } +sdist = { url = "https://files.pythonhosted.org/packages/75/a7/9810d872919697c9d01295633f5d574fb416d47e535f258272ca1f01f447/snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895", size = 105575, upload-time = "2025-05-09T16:34:51.843Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl", hash = "sha256:7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752", size = 104164, upload-time = "2026-06-03T00:56:38.614Z" }, + { url = "https://files.pythonhosted.org/packages/c8/78/3565d011c61f5a43488987ee32b6f3f656e7f107ac2782dd57bdd7d91d9a/snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064", size = 103274, upload-time = "2025-05-09T16:34:50.371Z" }, ] [[package]] @@ -1609,24 +1524,24 @@ wheels = [ [[package]] name = "types-pyyaml" -version = "6.0.12.20260518" +version = "6.0.12.20260408" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b8/83/4a1afc3fbfcf5b8d46fc390cd95ed6b0dc9010a265f4e9f46314efffa37a/types_pyyaml-6.0.12.20260518.tar.gz", hash = "sha256:d917f83fb38462550338c1297faedd860b3ec83912b96b1e3d73255f7473e466", size = 17850, upload-time = "2026-05-18T06:01:58.675Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/73/b759b1e413c31034cc01ecdfb96b38115d0ab4db55a752a3929f0cd449fd/types_pyyaml-6.0.12.20260408.tar.gz", hash = "sha256:92a73f2b8d7f39ef392a38131f76b970f8c66e4c42b3125ae872b7c93b556307", size = 17735, upload-time = "2026-04-08T04:30:50.974Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/06/a2/c01db32be2ae7d6a1689972f3c492b149ee4e164b12fdfd9f64b50888215/types_pyyaml-6.0.12.20260518-py3-none-any.whl", hash = "sha256:d2150f75a231c9fe9c7463bd29487d93e60bac90400287351384bc2284eba7cd", size = 20312, upload-time = "2026-05-18T06:01:57.368Z" }, + { url = "https://files.pythonhosted.org/packages/1c/f0/c391068b86abb708882c6d75a08cd7d25b2c7227dab527b3a3685a3c635b/types_pyyaml-6.0.12.20260408-py3-none-any.whl", hash = "sha256:fbc42037d12159d9c801ebfcc79ebd28335a7c13b08a4cfbc6916df78fee9384", size = 20339, upload-time = "2026-04-08T04:30:50.113Z" }, ] [[package]] name = "types-shapely" -version = "2.1.0.20260603" +version = "2.1.0.20260408" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/8d/b1ee75d100855ce214788b2e05b3648db50d6f0ff110fa986328a25e1fa1/types_shapely-2.1.0.20260603.tar.gz", hash = "sha256:0373ef71caa8c17aff0b810b8155f744ceaeee802f6f1438d09dd1c919612180", size = 27246, upload-time = "2026-06-03T06:42:05.798Z" } +sdist = { url = "https://files.pythonhosted.org/packages/10/8d/bf9e3eb51249601e22d797481999a06fb34998c4db5c76804394f8a3fa28/types_shapely-2.1.0.20260408.tar.gz", hash = "sha256:8552549d9429baa52ec4331e43b5db3b334fc3a7f30da48663010b7454b1451c", size = 26529, upload-time = "2026-04-08T04:34:42.111Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/10/e8/dfb9d2f832260d64d3b0b603ac6382ccc9b1a332535e434e9d5f97f7aed3/types_shapely-2.1.0.20260603-py3-none-any.whl", hash = "sha256:bc9ea51658b05db3b92e453cc74614af1d70eb391e4c36e15d80567fea6a61a8", size = 38338, upload-time = "2026-06-03T06:42:04.732Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3d/cbec691f56e71636192a07bf6809f598bed06d869b03b4e2b1ad2f7df032/types_shapely-2.1.0.20260408-py3-none-any.whl", hash = "sha256:8a31e2b074342a363f0c9d0c7d6e1e6c0dcce302a92ef94d64d0ca2a2b94a1d1", size = 37818, upload-time = "2026-04-08T04:34:41.243Z" }, ] [[package]]