diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/union_extraction.py b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/union_extraction.py index 45c8caca8..20418f771 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/union_extraction.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/extraction/union_extraction.py @@ -97,7 +97,7 @@ def extract_discriminator( _TypeShape = tuple[object, ...] -_FieldKey = tuple[str, _TypeShape] +_FieldKey = tuple[str, _TypeShape, frozenset[object]] def _structural_fingerprint(spec: FieldSpec) -> _TypeShape: @@ -231,16 +231,20 @@ def extract_union( for fs in member.spec.fields: if fs.name in shared_field_names: continue - key = (fs.name, _structural_fingerprint(fs)) + # The key includes the constraints fingerprint alongside the + # structural one: two arms with the same name and shape but + # different constraints (e.g. VehicleAxleCountSelector's + # `ge=1 + IntegerConstraint` vs the other selectors' `ge=0`) + # must not collapse into one `AnnotatedField` sharing a single + # constraint set -- that would silently drop one arm's rules. + # Keeping them as separate rows, each gated to its own + # `variant_sources`, reuses the same per-arm `Guard` mechanism + # that already handles a field present on only some arms + # (`check_builder._field_checks_for_union`), and the renderer's + # collision resolver already disambiguates multiple `Check`s + # landing on the same field label. + key = (fs.name, _structural_fingerprint(fs), _constraints_fingerprint(fs)) existing = seen.get(key) - if existing is not None: - existing_constraints = _constraints_fingerprint(existing.field_spec) - if _constraints_fingerprint(fs) != existing_constraints: - raise ValueError( - f"Union {name!r} field {fs.name!r} has the same structural " - f"shape across members but diverging constraints; dedup " - f"would silently drop one member's constraints" - ) prior_sources = existing.variant_sources or () if existing else () seen[key] = AnnotatedField( field_spec=fs, diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_render_common.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_render_common.py index bfa76ca0b..0fb881a1c 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_render_common.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/_render_common.py @@ -33,7 +33,7 @@ from overture.schema.system.field_path import ArrayPath, MapProjection -from .check_ir import Check, ModelCheck +from .check_ir import Check, Guard, ModelCheck from .constraint_dispatch import ForbidIf, RequireIf, model_constraint_function __all__ = [ @@ -277,13 +277,58 @@ def _symmetric_label_suffixes(keys: list[_K]) -> list[str]: return [f"_{idx}" if total > 1 else "" for idx, total in _occurrence_indices(keys)] +def _field_label_suffixes( + keys: list[tuple[str, str, tuple[Guard, ...]]], +) -> list[str]: + """Per-row field-label collision suffixes. + + `keys` carries `(base_label, check_name, guards)` per emitted row. A + field label collides for two reasons, resolved differently: + + - Across union arms -- the same field appears in several discriminator + arms, each carrying its own guard tuple. Every check gated to one + arm shares that arm's `_N` suffix (`N` its first-appearance order + among the label's arms), so a split field reports one consistent + label per arm. This includes a check unique to one arm (the axle + arm's `integer` check, absent from the dimension arms), which would + otherwise escape suffixing and report the bare label alongside its + `_N`-suffixed siblings. + - Within a single arm -- one field carries two same-named checks (a + lower- and upper-`bounds` pair emitted as separate checks). These + take a per-occurrence `_N` suffix keyed on `(label, name)`; a field + whose check names are all distinct stays bare. + + A label reached by a single arm uses the occurrence rule (leaving + unsplit fields untouched); a label reached by several uses the arm + rule. + """ + arms_by_label: dict[str, list[tuple[Guard, ...]]] = {} + for label, _name, guards in keys: + arms = arms_by_label.setdefault(label, []) + if guards not in arms: + arms.append(guards) + occurrences = _occurrence_indices([(label, name) for label, name, _ in keys]) + suffixes: list[str] = [] + for (label, _name, guards), (occ_idx, occ_total) in zip( + keys, occurrences, strict=True + ): + arms = arms_by_label[label] + if len(arms) > 1: + suffixes.append(f"_{arms.index(guards)}") + elif occ_total > 1: + suffixes.append(f"_{occ_idx}") + else: + suffixes.append("") + return suffixes + + @dataclass(frozen=True, slots=True) class FieldCheckRow: """One emitted field-check row, with its final derived strings. The renderer emits one row per descriptor of each `Check`. `field_check_rows` flattens the check list into these rows once, - computing both the symmetric `label` collision suffix and the + computing both the arm-grouped `label` collision suffix and the asymmetric `func_name` disambiguation, so the renderer and test renderer agree without each re-deriving them by a positional index. @@ -341,15 +386,25 @@ def field_check_rows(field_checks: list[Check]) -> list[FieldCheckRow]: raw_func_names.append(f"_{sanitize_field_name(label)}{func_suffix}_check") flattened.append((check, desc_idx, label, name)) func_names = disambiguate(raw_func_names) - label_suffixes = _symmetric_label_suffixes( - [(label, name) for _check, _idx, label, name in flattened] + label_suffixes = _field_label_suffixes( + [(label, name, check.guards) for check, _idx, label, name in flattened] ) - return [ + rows = [ FieldCheckRow(check, desc_idx, f"{label}{label_suffix}", name, func_name) for (check, desc_idx, label, name), label_suffix, func_name in zip( flattened, label_suffixes, func_names, strict=True ) ] + # Arm-grouped suffixing cannot distinguish two same-name checks that + # land in one arm of a split field; fail generation loudly if a schema + # ever produces that instead of emitting indistinguishable violations. + identities = [(row.label, row.name) for row in rows] + duplicates = {i for i in identities if identities.count(i) > 1} + if duplicates: + raise ValueError( + f"Duplicate violation identities in generated checks: {sorted(duplicates)}" + ) + return rows @dataclass(frozen=True, slots=True) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/constraint_dispatch.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/constraint_dispatch.py index be3008303..529ac543e 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/constraint_dispatch.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/constraint_dispatch.py @@ -18,6 +18,7 @@ from overture.schema.system.case import to_snake_case from overture.schema.system.field_constraint.collection import UniqueItemsConstraint +from overture.schema.system.field_constraint.numeric import IntegerConstraint from overture.schema.system.field_constraint.string import ( JsonPointerConstraint, PatternConstraint, @@ -350,6 +351,12 @@ def _raw_pattern(constraint: object) -> str | None: function="check_geometry_type", args=tuple(c.allowed_types) ), ), + ( + IntegerConstraint, + lambda c, _bt: ExpressionDescriptor( + function="check_integer", constraint_type=type(c) + ), + ), ] diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/schema_builder.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/schema_builder.py index 08469d67c..27b98be55 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/schema_builder.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/schema_builder.py @@ -16,7 +16,7 @@ Scalar, UnionRef, ) -from ..extraction.field_walk import enum_source, terminal_scalar +from ..extraction.field_walk import enum_source from ..extraction.specs import FieldSpec, ModelSpec, UnionSpec from ..extraction.type_registry import get_type_mapping @@ -24,7 +24,6 @@ "SHARED_TYPE_REFS", "SchemaField", "build_schema", - "spark_type_rank", ] # Types whose base_type name maps to a _schema_structs.py StructType constant. @@ -87,38 +86,21 @@ def _spark_for_scalar(scalar: Scalar) -> str: return _spark_for_base(scalar.base_type, scalar.source_type) -# Spark numeric type widening precedence (higher rank = wider type). -_SPARK_TYPE_WIDENING: dict[str, int] = { - "IntegerType()": 0, - "LongType()": 1, - "DoubleType()": 2, -} - - -def spark_type_rank(field_spec: FieldSpec) -> int: - """Return a widening rank for the field's resolved Spark type. - - Fields with a higher rank are preferred when deduplicating union - members by name. Non-numeric types return -1 (no widening). - """ - scalar = terminal_scalar(field_spec.shape) - if not isinstance(scalar, Primitive): - return -1 - expr = _spark_for_base(scalar.base_type, scalar.source_type) - return _SPARK_TYPE_WIDENING.get(expr, -1) - - def _deduplicate_by_name(fields: list[FieldSpec]) -> list[FieldSpec]: - """Keep one FieldSpec per name, widening the Spark type on conflict. - - Union annotated_fields may contain the same field name with different - type shapes (e.g. `value` as uint8 in one variant and float64 in - another). Parquet stores one column per name, so the schema needs - exactly one entry. When two fields share a name, the one with the - wider numeric Spark type wins (matching Parquet's type-widening - behavior). Two same-named fields whose Spark types are non-numeric and - not identical cannot share a column, so the collision fails loudly - rather than silently keeping whichever arm came first. + """Keep one FieldSpec per name, requiring every arm to agree on Spark type. + + Union annotated_fields may contain the same field name declared by + multiple arms with different `FieldSpec`s -- a `Literal` discriminator + whose value differs per arm, or a field whose per-arm constraints + diverge (see `union_extraction.extract_union`). Parquet stores one + column per name, so the schema needs exactly one entry. Two same-named + fields are compatible when they resolve to the SAME Spark type -- the + first-seen `FieldSpec`'s shape is kept (arbitrarily; the column type is + identical either way). Two same-named fields that resolve to DIFFERENT + Spark types cannot share a column: this always raises, whether the + mismatch is numeric (a narrower int type vs a float) or not, rather + than silently widening to the broader type -- that widening is exactly + the implicit Parquet-compatibility trap this check exists to surface. """ seen: dict[str, FieldSpec] = {} for f in fields: @@ -126,19 +108,16 @@ def _deduplicate_by_name(fields: list[FieldSpec]) -> list[FieldSpec]: if existing is None: seen[f.name] = f continue - rank_f, rank_existing = spark_type_rank(f), spark_type_rank(existing) - if rank_f < 0 and rank_existing < 0: - spark_f = _shape_to_spark(f.shape) - spark_existing = _shape_to_spark(existing.shape) - if spark_f != spark_existing: - raise ValueError( - f"Union field {f.name!r} resolves to incompatible " - f"non-widening Spark types across arms " - f"({spark_existing} vs {spark_f}); a single Parquet " - "column cannot represent both." - ) - if rank_f > rank_existing: - seen[f.name] = f + spark_f, spark_existing = ( + _shape_to_spark(f.shape), + _shape_to_spark(existing.shape), + ) + if spark_f != spark_existing: + raise ValueError( + f"Union field {f.name!r} resolves to incompatible Spark " + f"types across arms ({spark_existing} vs {spark_f}); a " + "single Parquet column cannot represent both." + ) return list(seen.values()) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/base_row.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/base_row.py index 70a20e0ee..ab4715a34 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/base_row.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/base_row.py @@ -58,7 +58,6 @@ require_bool_field_eq, require_field_eq, ) -from ..schema_builder import spark_type_rank from .constraint_values import ( CONSTRAINT_VALUES, curated_pattern_values, @@ -488,27 +487,16 @@ def value_for_field( ) -def _widest_union_member(union: UnionSpec) -> RecordSpec: - """Pick the union member whose fields have the highest cumulative Spark type rank. +def _default_union_member(union: UnionSpec) -> RecordSpec: + """Return the union member used when no discriminator value is known. - When multiple union members share a field name with different numeric - types (e.g. `value: uint8` in one variant and `value: float64` in - another), PySpark widens the column to the broadest type (DoubleType). - Generating a row from the narrower member produces Python `int` values - that PySpark silently converts to null in `DoubleType` columns. - - By selecting the member with the widest field types, the generated row - uses Python `float` values that PySpark accepts in `DoubleType` columns. + A field shared by name across arms always resolves to the same Spark + type in every arm (enforced by `schema_builder._deduplicate_by_name`, + which raises otherwise), so any arm's synthesized value is safe to + write into that shared column -- the member choice is arbitrary. Picks + the first member, deterministically, so regeneration is stable. """ - best_spec = union.member_specs[0].spec - best_rank = -1 - for member in union.member_specs: - field_ranks = [spark_type_rank(f) for f in member.spec.fields] - rank = sum(r for r in field_ranks if r >= 0) - if rank > best_rank: - best_rank = rank - best_spec = member.spec - return best_spec + return union.member_specs[0].spec def resolve_arm_spec( @@ -517,11 +505,12 @@ def resolve_arm_spec( """Return the member `RecordSpec` for one arm of a discriminated union. Without a discriminator value (a check not gated to a specific arm), - returns the widest member -- the one whose float types survive PySpark - column widening, per `_widest_union_member`. With a value, returns the - member that value selects, and raises when it selects none: a seeded - discriminator that matches no arm is a check_builder/scaffold inconsistency, - not a reason to fall back to an arm whose fields contradict the seed. + returns the union's first member -- safe because a field shared by name + across arms always resolves to the same Spark type (see + `_default_union_member`). With a value, returns the member that value + selects, and raises when it selects none: a seeded discriminator that + matches no arm is a check_builder/scaffold inconsistency, not a reason + to fall back to an arm whose fields contradict the seed. Parameters ---------- @@ -537,7 +526,7 @@ def resolve_arm_spec( When `discriminator_value` is given but selects no member arm. """ if discriminator_value is None: - return _widest_union_member(union) + return _default_union_member(union) mapping = union.discriminator_mapping or {} member_cls = mapping.get(discriminator_value) # type: ignore[call-overload] if member_cls is not None: @@ -645,7 +634,7 @@ def _value_for_shape( # the sparse case the field is omitted from the dict and Pydantic # supplies the default during `TypeAdapter.validate_python()`. return _row_from_model_spec( - _widest_union_member(u), + _default_union_member(u), index=index, populate_optional=populate_optional, ) diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/invalid_value.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/invalid_value.py index e21812a84..9ab170c09 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/invalid_value.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/invalid_value.py @@ -33,6 +33,12 @@ _INVALID_LITERALS: dict[str, object] = { "check_required": None, "check_enum": "__INVALID__", + # Non-integral, and >= 1 so an accompanying `ge=1` bound (the only + # IntegerConstraint field today, axle count) does not also fire -- the + # scenario isolates the integer check. A future integer field with a + # higher lower bound would additionally trip its bounds check, which the + # "at least the expected check fires" harness still tolerates. + "check_integer": 1.5, "check_url_format": "not-a-url", "check_url_length": "https://" + "x" * 2076, "check_email": "not-an-email", diff --git a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/scaffold.py b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/scaffold.py index 3d8d6554e..4d48dde05 100644 --- a/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/scaffold.py +++ b/packages/overture-schema-codegen/src/overture/schema/codegen/pyspark/test_data/scaffold.py @@ -116,7 +116,7 @@ def _child_container_spec( """Resolve the model a path field descends into. Returns the field's terminal `ModelRef` model, or -- for a discriminated - union -- the member arm the `discriminator_value` selects (the widest + union -- the member arm the `discriminator_value` selects (the first member when the check is not arm-gated). `None` when the field has neither a model nor a union terminal. """ @@ -144,7 +144,7 @@ def _walk_to_target( (`generate_base_row` -- required fields populated and model constraints such as `require_any_of` satisfied), then the on-path child overrides its field. A discriminated-union element resolves to the arm the seeded - discriminator selects (or the widest member when the check is not + discriminator selects (or the first member when the check is not arm-gated), so the element is a valid instance of a concrete arm rather than an untagged `{}`. diff --git a/packages/overture-schema-codegen/tests/test_constraint_description.py b/packages/overture-schema-codegen/tests/test_constraint_description.py index 941537a2e..4d15f6a2e 100644 --- a/packages/overture-schema-codegen/tests/test_constraint_description.py +++ b/packages/overture-schema-codegen/tests/test_constraint_description.py @@ -21,6 +21,7 @@ ) from overture.schema.codegen.extraction.specs import TypeIdentity from overture.schema.codegen.extraction.type_analyzer import ConstraintSource +from overture.schema.system.field_constraint.numeric import IntegerConstraint from overture.schema.system.field_constraint.string import ( CountryCodeAlpha2Constraint, PatternConstraint, @@ -544,6 +545,15 @@ def link_fn(tid: TypeIdentity) -> str: assert received[0].obj is Target assert result == "References [`Target`](link) (composition)" + def test_integer_constraint_renders_docstring_and_class(self) -> None: + """IntegerConstraint (opaque, docstringed) flows through to prose.""" + cs = ConstraintSource( + source_ref=None, source_name=None, constraint=IntegerConstraint() + ) + result = constraint_display_text(cs) + assert "integral" in result + assert "`IntegerConstraint`" in result + class TestConstraintPatternFlags: """constraint_display_text surfaces a compiled pattern's regex flags.""" diff --git a/packages/overture-schema-codegen/tests/test_pyspark_base_row.py b/packages/overture-schema-codegen/tests/test_pyspark_base_row.py index a1e9ffb94..ce8b2116b 100644 --- a/packages/overture-schema-codegen/tests/test_pyspark_base_row.py +++ b/packages/overture-schema-codegen/tests/test_pyspark_base_row.py @@ -5,7 +5,7 @@ from typing import Any import pytest -from annotated_types import Gt, Lt +from annotated_types import Ge, Gt, Lt from codegen_test_support import ( FeatureWithDict, FeatureWithRequiredUrl, @@ -27,6 +27,7 @@ ModelSpec, UnionSpec, ) +from overture.schema.codegen.extraction.union_extraction import extract_union from overture.schema.codegen.pyspark.constraint_dispatch import ExpressionDescriptor from overture.schema.codegen.pyspark.test_data.base_row import ( _primitive_default, @@ -39,8 +40,11 @@ generate_base_row, generate_populated_arm_rows, generate_populated_row, + resolve_arm_spec, value_for_field, ) +from overture.schema.common.scoping.vehicle import VehicleSelector +from overture.schema.system.field_constraint.numeric import IntegerConstraint from overture.schema.system.model_constraint import ( FieldEqCondition, forbid_if, @@ -133,6 +137,31 @@ def test_geometry_is_valid_wkt(self, connector_spec: ModelSpec) -> None: assert row["geometry"].startswith("POINT") +class TestResolveArmSpecNoDiscriminatorValue: + """Without a discriminator value, resolve_arm_spec falls back deterministically. + + Pre-widening-removal, the fallback picked the "widest" numeric member so a + narrower arm's int value wouldn't null out in a widened DoubleType column. + Now that `schema_builder._deduplicate_by_name` requires every arm to + resolve to the SAME Spark type for a shared field name, any arm's + synthesized value fits the shared column, so the fallback is just "first + member" -- exercised here via the real VehicleSelector union (five arms, + `value` now float64 on every arm). + """ + + def test_returns_first_member_deterministically(self) -> None: + spec = extract_union("VehicleAxleCountSelector", VehicleSelector) + result = resolve_arm_spec(spec) + assert result is spec.member_specs[0].spec + + def test_value_for_shape_produces_valid_row_from_fallback_arm(self) -> None: + """The fallback arm's synthesized row validates against the union.""" + spec = extract_union("VehicleAxleCountSelector", VehicleSelector) + row = generate_base_row(resolve_arm_spec(spec)) + adapter: TypeAdapter[object] = TypeAdapter(spec.source_annotation) + adapter.validate_python(row) + + class TestGenerateArmRows: def test_returns_dict_per_arm( self, segment_spec: ModelSpec, segment_union: UnionSpec @@ -587,3 +616,19 @@ def test_gt_and_lt_float_tight_interval_returns_interior_value(self) -> None: result = _value_from_scalar_constraints(scalar) assert isinstance(result, float) assert 0.0 < result < 1.0 + + def test_integer_constraint_with_ge_returns_integral_value(self) -> None: + """A float64 IntegerConstraint+ge=1 field synthesizes an integral value >= 1.""" + scalar = Primitive( + base_type="float64", + constraints=( + ConstraintSource( + source_ref=None, source_name=None, constraint=IntegerConstraint() + ), + ConstraintSource(source_ref=None, source_name=None, constraint=Ge(1)), + ), + ) + result = _value_from_scalar_constraints(scalar) + assert isinstance(result, float) + assert result >= 1 + assert result.is_integer() diff --git a/packages/overture-schema-codegen/tests/test_pyspark_constraint_dispatch.py b/packages/overture-schema-codegen/tests/test_pyspark_constraint_dispatch.py index 765fdbb29..7a6ad4a4a 100644 --- a/packages/overture-schema-codegen/tests/test_pyspark_constraint_dispatch.py +++ b/packages/overture-schema-codegen/tests/test_pyspark_constraint_dispatch.py @@ -30,6 +30,7 @@ normalize_anchor, ) from overture.schema.system.field_constraint.collection import UniqueItemsConstraint +from overture.schema.system.field_constraint.numeric import IntegerConstraint from overture.schema.system.field_constraint.string import ( CountryCodeAlpha2Constraint, JsonPointerConstraint, @@ -313,6 +314,12 @@ def test_geometry_type(self) -> None: assert desc.function == "check_geometry_type" assert GeometryType.POINT in desc.args + def test_integer_constraint(self) -> None: + desc = dispatch_constraint(IntegerConstraint(), base_type="float64") + assert desc is not None + assert desc.function == "check_integer" + assert desc.constraint_type is IntegerConstraint + class TestSkippedConstraints: def test_reference_returns_none(self) -> None: diff --git a/packages/overture-schema-codegen/tests/test_pyspark_invalid_value.py b/packages/overture-schema-codegen/tests/test_pyspark_invalid_value.py index 9fceb373f..4f31b6594 100644 --- a/packages/overture-schema-codegen/tests/test_pyspark_invalid_value.py +++ b/packages/overture-schema-codegen/tests/test_pyspark_invalid_value.py @@ -53,6 +53,14 @@ def test_unknown_bound_raises(self) -> None: invalid_value(desc) +class TestInvalidValueInteger: + def test_returns_non_integral_float(self) -> None: + desc = ExpressionDescriptor(function="check_integer") + value = invalid_value(desc) + assert isinstance(value, float) + assert not value.is_integer() + + class TestInvalidValuePattern: def test_unknown_constraint_type_raises(self) -> None: desc = ExpressionDescriptor(function="check_pattern", args=(r"^[A-Z]+$",)) diff --git a/packages/overture-schema-codegen/tests/test_pyspark_renderer.py b/packages/overture-schema-codegen/tests/test_pyspark_renderer.py index 27f0dd396..42d2d7305 100644 --- a/packages/overture-schema-codegen/tests/test_pyspark_renderer.py +++ b/packages/overture-schema-codegen/tests/test_pyspark_renderer.py @@ -1125,6 +1125,42 @@ def test_labels_are_positional_not_identity_keyed(self) -> None: labels = [row.label for row in field_check_rows([first, second, first_copy])] assert labels == ["value_0", "value_1", "value_2"], labels + def test_arm_unique_check_shares_arm_suffix(self) -> None: + """A check present in only one arm takes that arm's suffix, not the bare label. + + The axle arm carries an `integer` check the dimension arms lack. + Keyed on `(field, name)` the lone `integer` row is unique, so it + would escape suffixing and report the bare `value` beside its + `value_0` siblings. Grouping by arm keeps the whole axle arm on + `value_0`. + """ + axle = (ElementGuard(discriminator="dimension", values=("axle_count",)),) + dimension = ( + ElementGuard(discriminator="dimension", values=("height", "width")), + ) + axle_required = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("value"), + guards=axle, + ) + axle_integer = Check( + descriptors=(ExpressionDescriptor(function="check_integer"),), + target=_path("value"), + guards=axle, + ) + dimension_required = Check( + descriptors=(ExpressionDescriptor(function="check_required"),), + target=_path("value"), + guards=dimension, + ) + rows = field_check_rows([axle_required, axle_integer, dimension_required]) + labeled = {(row.name, row.label) for row in rows} + assert labeled == { + ("required", "value_0"), + ("integer", "value_0"), + ("required", "value_1"), + }, labeled + class TestMapPathRendering: """MapPath targets render to map_keys_check / map_values_check.""" diff --git a/packages/overture-schema-codegen/tests/test_pyspark_schema_builder.py b/packages/overture-schema-codegen/tests/test_pyspark_schema_builder.py index c848d88b4..403109bd9 100644 --- a/packages/overture-schema-codegen/tests/test_pyspark_schema_builder.py +++ b/packages/overture-schema-codegen/tests/test_pyspark_schema_builder.py @@ -211,3 +211,61 @@ def test_one_schema_field_per_name(self, fields: list[SchemaField]) -> None: def test_color_field_is_string_type(self, fields: list[SchemaField]) -> None: color_field = next(f for f in fields if f.name == "color") assert color_field.type_expr == "StringType()" + + +class _NumberA(BaseModel): + pass + + +class _NumberB(BaseModel): + pass + + +class TestUnionSchemaTypeDivergence: + """build_schema raises on ANY resolved-Spark-type divergence, always -- + no numeric widening. A collision where both sides resolve to the same + Spark type (e.g. two distinct scalar types, both IntegerType) still + succeeds.""" + + def _fields_for(self, shape_a: Primitive, shape_b: Primitive) -> list[SchemaField]: + af_a = AnnotatedField( + field_spec=FieldSpec(name="value", shape=shape_a, is_required=True), + variant_sources=(_NumberA,), + ) + af_b = AnnotatedField( + field_spec=FieldSpec(name="value", shape=shape_b, is_required=True), + variant_sources=(_NumberB,), + ) + spec = UnionSpec( + name="TestNumberUnion", + description=None, + annotated_fields=[af_a, af_b], + members=[], + discriminator_field=None, + discriminator_mapping=None, + source_annotation=object(), + common_base=BaseModel, + ) + return build_schema(spec) + + def test_numeric_widening_now_raises(self) -> None: + """uint8 (IntegerType) vs float64 (DoubleType) previously widened + silently; it now raises like any other type mismatch.""" + with pytest.raises(ValueError, match="incompatible"): + self._fields_for( + Primitive(base_type="uint8"), Primitive(base_type="float64") + ) + + def test_non_numeric_mismatch_still_raises(self) -> None: + with pytest.raises(ValueError, match="incompatible"): + self._fields_for(Primitive(base_type="str"), Primitive(base_type="bool")) + + def test_same_resolved_type_does_not_raise(self) -> None: + """Two distinct scalar types resolving to the same Spark type + (e.g. int32 and uint16, both IntegerType) share a column cleanly.""" + fields = self._fields_for( + Primitive(base_type="int32"), Primitive(base_type="uint16") + ) + value_fields = [f for f in fields if f.name == "value"] + assert len(value_fields) == 1 + assert value_fields[0].type_expr == "IntegerType()" diff --git a/packages/overture-schema-codegen/tests/test_union_extraction.py b/packages/overture-schema-codegen/tests/test_union_extraction.py index 73f599332..a543c63c4 100644 --- a/packages/overture-schema-codegen/tests/test_union_extraction.py +++ b/packages/overture-schema-codegen/tests/test_union_extraction.py @@ -6,9 +6,11 @@ import pytest from annotated_types import MinLen from codegen_test_support import ( + LongNamesSegment, RailSegment, RoadSegment, SegmentBase, + ShortNamesSegment, TestEnumDiscriminatorUnion, TestSegment, TestSegmentDivergingConstraints, @@ -20,6 +22,7 @@ ConstraintSource, Primitive, ) +from overture.schema.codegen.extraction.length_constraints import ArrayMinLen from overture.schema.codegen.extraction.specs import FieldSpec, UnionSpec from overture.schema.codegen.extraction.union_extraction import ( _constraints_fingerprint, @@ -132,21 +135,50 @@ def test_discriminator_mapping_uses_enum_values(self, spec: UnionSpec) -> None: class TestDivergingConstraints: - """Same-named fields with matching shape but diverging constraints fail loudly.""" + """Same-named fields with matching shape but diverging constraints split + into separate arm-gated `AnnotatedField`s rather than raising. + + Field-level checks are already arm-gated by `Guard`s built from + `variant_sources` (see `check_builder._field_checks_for_union`), and the + renderer's collision resolver already disambiguates multiple `Check`s + that land on the same field label (e.g. the pre-existing `value_0`/ + `value_1` split for a field required only on some arms). Keeping + diverging-constraint fields as separate rows reuses that machinery + instead of dropping one arm's constraints or refusing to extract. + """ + + def test_diverging_constraints_produce_separate_annotated_fields(self) -> None: + """`ShortNamesSegment` and `LongNamesSegment` both declare `aliases` + as `list[str] | None` -- structurally identical -- but their + `min_length` constraints differ (1 vs 5). Extraction keeps them as + two `AnnotatedField`s, each gated to the arm that declared it. + """ + spec = extract_union( + "TestSegmentDivergingConstraints", TestSegmentDivergingConstraints + ) + aliases_fields = [ + af for af in spec.annotated_fields if af.field_spec.name == "aliases" + ] + assert len(aliases_fields) == 2 - def test_diverging_constraints_raise(self) -> None: - """A field shared by structure but not by constraints raises ValueError. + by_source = {af.variant_sources: af for af in aliases_fields} + assert (ShortNamesSegment,) in by_source + assert (LongNamesSegment,) in by_source - `ShortNamesSegment` and `LongNamesSegment` both declare `aliases` - as `list[str] | None`, so the structural fingerprint collapses - them — but the `min_length` constraints differ. Dedup would - silently keep one member's `FieldSpec`, so extraction raises - instead. - """ - with pytest.raises(ValueError, match="diverging constraints"): - extract_union( - "TestSegmentDivergingConstraints", TestSegmentDivergingConstraints - ) + def min_length(af: object) -> int: + for cs in af.field_spec.shape.constraints: # type: ignore[attr-defined] + if isinstance(cs.constraint, ArrayMinLen): + return cs.constraint.min_length # type: ignore[no-any-return] + raise AssertionError("no ArrayMinLen constraint found") + + assert min_length(by_source[(ShortNamesSegment,)]) == 1 + assert min_length(by_source[(LongNamesSegment,)]) == 5 + + def test_diverging_constraints_do_not_raise(self) -> None: + """Extraction succeeds where it previously raised ValueError.""" + extract_union( + "TestSegmentDivergingConstraints", TestSegmentDivergingConstraints + ) class TestUnionNameDerivation: diff --git a/packages/overture-schema-common/src/overture/schema/common/scoping/vehicle.py b/packages/overture-schema-common/src/overture/schema/common/scoping/vehicle.py index 287b25c55..93e71fa11 100644 --- a/packages/overture-schema-common/src/overture/schema/common/scoping/vehicle.py +++ b/packages/overture-schema-common/src/overture/schema/common/scoping/vehicle.py @@ -8,8 +8,9 @@ from pydantic import BaseModel, Field from overture.schema.common.unit import LengthUnit, WeightUnit +from overture.schema.system.field_constraint import IntegerConstraint from overture.schema.system.model_constraint import no_extra_fields -from overture.schema.system.primitive import float64, uint8 +from overture.schema.system.primitive import float64 class VehicleDimension(str, Enum): @@ -54,7 +55,15 @@ class VehicleAxleCountSelector(VehicleSelectorBase): """Selects vehicles based on the number of axles they have.""" dimension: Literal[VehicleDimension.AXLE_COUNT] - value: uint8 = Field(description="Number of axles on the vehicle") + # A whole number stored as float64 so the flattened `value` column across + # all VehicleSelector arms is a single Parquet type. The bound precedes + # IntegerConstraint so pydantic attaches `ge` to the numeric schema + # (emitting JSON Schema `minimum`) rather than to the wrapping validator. + value: Annotated[ + float64, + Field(ge=1, description="Number of axles on the vehicle"), + IntegerConstraint(), + ] @no_extra_fields diff --git a/packages/overture-schema-common/tests/scoping/test_vehicle.py b/packages/overture-schema-common/tests/scoping/test_vehicle.py new file mode 100644 index 000000000..1146248a9 --- /dev/null +++ b/packages/overture-schema-common/tests/scoping/test_vehicle.py @@ -0,0 +1,51 @@ +"""Validation behavior for the vehicle selector subtypes.""" + +import pytest +from overture.schema.common.scoping.vehicle import ( + VehicleAxleCountSelector, + VehicleSelector, +) +from pydantic import TypeAdapter, ValidationError + +_adapter: TypeAdapter[object] = TypeAdapter(VehicleSelector) + + +class TestVehicleAxleCountSelector: + def test_integral_axle_count_accepted(self) -> None: + selector = VehicleAxleCountSelector( + dimension="axle_count", comparison="greater_than", value=2 + ) + assert selector.value == 2.0 + assert isinstance(selector.value, float) + + def test_minimum_one_axle_accepted(self) -> None: + selector = VehicleAxleCountSelector( + dimension="axle_count", comparison="equal", value=1 + ) + assert selector.value == 1.0 + + def test_zero_axles_rejected(self) -> None: + with pytest.raises(ValidationError): + VehicleAxleCountSelector( + dimension="axle_count", comparison="equal", value=0 + ) + + def test_fractional_axle_count_rejected(self) -> None: + with pytest.raises(ValidationError) as exc_info: + VehicleAxleCountSelector( + dimension="axle_count", comparison="equal", value=2.5 + ) + assert "integer" in str(exc_info.value).lower() + + def test_discriminated_union_routes_to_axle_count(self) -> None: + selector = _adapter.validate_python( + {"dimension": "axle_count", "comparison": "greater_than_equal", "value": 3} + ) + assert isinstance(selector, VehicleAxleCountSelector) + assert selector.value == 3.0 + + def test_discriminated_union_rejects_fractional_axle_count(self) -> None: + with pytest.raises(ValidationError): + _adapter.validate_python( + {"dimension": "axle_count", "comparison": "equal", "value": 1.5} + ) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/constraint_expressions.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/constraint_expressions.py index 52c422e2c..955c484b5 100644 --- a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/constraint_expressions.py +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/constraint_expressions.py @@ -116,6 +116,25 @@ def check_bounds( return F.coalesce(nan_check, *checks) +def check_integer(col: Column) -> Column: + """Integral-value check: a float column must hold a whole number. + + Fires when the value carries a fractional part (`2.5`), the case a + `float64` column standing in for a count must reject. Testing the + fractional part in double space (`col % 1 != 0`) matches Pydantic + exactly: large integral doubles like `1e30` -- whole numbers Pydantic + accepts ((1e30).is_integer() is True) -- pass, where a `floor`-based + check would saturate the LongType cast above 2^63 and wrongly fire. + NaN and infinity yield a NaN remainder (`!= 0`) and are rejected + directly, matching `.is_integer()` returning False for both. A null + passes (presence is `check_required`'s concern). + """ + return F.when( + col.isNotNull() & (col % 1 != 0), + error_msg("must be an integer, got ", col.cast("string")), + ) + + def check_enum( col: Column, allowed: list[str], diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/transportation/segment.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/transportation/segment.py index 5ef1d437e..a07af6c1a 100644 --- a/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/transportation/segment.py +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated/overture/schema/transportation/segment.py @@ -37,6 +37,7 @@ check_enum, check_forbid_if, check_geometry_type, + check_integer, check_json_pointer, check_linear_range_bounds, check_linear_range_length, @@ -764,7 +765,7 @@ def _access_restrictions_when_vehicle_comparison_enum_check() -> Check: ) -def _access_restrictions_when_vehicle_value_check() -> Check: +def _access_restrictions_when_vehicle_value_required_check() -> Check: return Check( field="access_restrictions[].when.vehicle[].value_0", name="required", @@ -783,7 +784,45 @@ def _access_restrictions_when_vehicle_value_check() -> Check: ) -def _access_restrictions_when_vehicle_value_required_check() -> Check: +def _access_restrictions_when_vehicle_value_bounds_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].value_0", + name="bounds", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["axle_count"]), + check_bounds(inner["value"], ge=1.0), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_value_integer_check() -> Check: + return Check( + field="access_restrictions[].when.vehicle[].value_0", + name="integer", + expr=nested_array_check( + "access_restrictions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["axle_count"]), + check_integer(inner["value"]), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"access_restrictions"}), + ) + + +def _access_restrictions_when_vehicle_value_required_check_1() -> Check: return Check( field="access_restrictions[].when.vehicle[].value_1", name="required", @@ -802,9 +841,9 @@ def _access_restrictions_when_vehicle_value_required_check() -> Check: ) -def _access_restrictions_when_vehicle_value_bounds_check() -> Check: +def _access_restrictions_when_vehicle_value_bounds_check_1() -> Check: return Check( - field="access_restrictions[].when.vehicle[].value", + field="access_restrictions[].when.vehicle[].value_1", name="bounds", expr=nested_array_check( "access_restrictions", @@ -2502,7 +2541,7 @@ def _prohibited_transitions_when_vehicle_comparison_enum_check() -> Check: ) -def _prohibited_transitions_when_vehicle_value_check() -> Check: +def _prohibited_transitions_when_vehicle_value_required_check() -> Check: return Check( field="prohibited_transitions[].when.vehicle[].value_0", name="required", @@ -2524,7 +2563,51 @@ def _prohibited_transitions_when_vehicle_value_check() -> Check: ) -def _prohibited_transitions_when_vehicle_value_required_check() -> Check: +def _prohibited_transitions_when_vehicle_value_bounds_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].value_0", + name="bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["axle_count"]), + check_bounds(inner["value"], ge=1.0), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_value_integer_check() -> Check: + return Check( + field="prohibited_transitions[].when.vehicle[].value_0", + name="integer", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "prohibited_transitions", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["axle_count"]), + check_integer(inner["value"]), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"prohibited_transitions", "subtype"}), + ) + + +def _prohibited_transitions_when_vehicle_value_required_check_1() -> Check: return Check( field="prohibited_transitions[].when.vehicle[].value_1", name="required", @@ -2548,9 +2631,9 @@ def _prohibited_transitions_when_vehicle_value_required_check() -> Check: ) -def _prohibited_transitions_when_vehicle_value_bounds_check() -> Check: +def _prohibited_transitions_when_vehicle_value_bounds_check_1() -> Check: return Check( - field="prohibited_transitions[].when.vehicle[].value", + field="prohibited_transitions[].when.vehicle[].value_1", name="bounds", expr=F.when( F.col("subtype").isin(["road"]), @@ -3480,7 +3563,7 @@ def _speed_limits_when_vehicle_comparison_enum_check() -> Check: ) -def _speed_limits_when_vehicle_value_check() -> Check: +def _speed_limits_when_vehicle_value_required_check() -> Check: return Check( field="speed_limits[].when.vehicle[].value_0", name="required", @@ -3502,7 +3585,51 @@ def _speed_limits_when_vehicle_value_check() -> Check: ) -def _speed_limits_when_vehicle_value_required_check() -> Check: +def _speed_limits_when_vehicle_value_bounds_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].value_0", + name="bounds", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["axle_count"]), + check_bounds(inner["value"], ge=1.0), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_value_integer_check() -> Check: + return Check( + field="speed_limits[].when.vehicle[].value_0", + name="integer", + expr=F.when( + F.col("subtype").isin(["road"]), + nested_array_check( + "speed_limits", + lambda el: array_check( + el["when"]["vehicle"], + lambda inner: F.when( + inner["dimension"].isin(["axle_count"]), + check_integer(inner["value"]), + ), + ), + ), + ), + shape=CheckShape.ARRAY, + read_columns=frozenset({"speed_limits", "subtype"}), + ) + + +def _speed_limits_when_vehicle_value_required_check_1() -> Check: return Check( field="speed_limits[].when.vehicle[].value_1", name="required", @@ -3526,9 +3653,9 @@ def _speed_limits_when_vehicle_value_required_check() -> Check: ) -def _speed_limits_when_vehicle_value_bounds_check() -> Check: +def _speed_limits_when_vehicle_value_bounds_check_1() -> Check: return Check( - field="speed_limits[].when.vehicle[].value", + field="speed_limits[].when.vehicle[].value_1", name="bounds", expr=F.when( F.col("subtype").isin(["road"]), @@ -4535,9 +4662,11 @@ def segment_checks() -> list[Check]: _access_restrictions_when_vehicle_dimension_enum_check(), _access_restrictions_when_vehicle_comparison_required_check(), _access_restrictions_when_vehicle_comparison_enum_check(), - _access_restrictions_when_vehicle_value_check(), _access_restrictions_when_vehicle_value_required_check(), _access_restrictions_when_vehicle_value_bounds_check(), + _access_restrictions_when_vehicle_value_integer_check(), + _access_restrictions_when_vehicle_value_required_check_1(), + _access_restrictions_when_vehicle_value_bounds_check_1(), _access_restrictions_when_vehicle_unit_required_check(), _access_restrictions_when_vehicle_unit_enum_check(), _access_restrictions_when_vehicle_unit_required_check_1(), @@ -4645,9 +4774,11 @@ def segment_checks() -> list[Check]: _prohibited_transitions_when_vehicle_dimension_enum_check(), _prohibited_transitions_when_vehicle_comparison_required_check(), _prohibited_transitions_when_vehicle_comparison_enum_check(), - _prohibited_transitions_when_vehicle_value_check(), _prohibited_transitions_when_vehicle_value_required_check(), _prohibited_transitions_when_vehicle_value_bounds_check(), + _prohibited_transitions_when_vehicle_value_integer_check(), + _prohibited_transitions_when_vehicle_value_required_check_1(), + _prohibited_transitions_when_vehicle_value_bounds_check_1(), _prohibited_transitions_when_vehicle_unit_required_check(), _prohibited_transitions_when_vehicle_unit_enum_check(), _prohibited_transitions_when_vehicle_unit_required_check_1(), @@ -4699,9 +4830,11 @@ def segment_checks() -> list[Check]: _speed_limits_when_vehicle_dimension_enum_check(), _speed_limits_when_vehicle_comparison_required_check(), _speed_limits_when_vehicle_comparison_enum_check(), - _speed_limits_when_vehicle_value_check(), _speed_limits_when_vehicle_value_required_check(), _speed_limits_when_vehicle_value_bounds_check(), + _speed_limits_when_vehicle_value_integer_check(), + _speed_limits_when_vehicle_value_required_check_1(), + _speed_limits_when_vehicle_value_bounds_check_1(), _speed_limits_when_vehicle_unit_required_check(), _speed_limits_when_vehicle_unit_enum_check(), _speed_limits_when_vehicle_unit_required_check_1(), diff --git a/packages/overture-schema-pyspark/tests/expressions/test_constraint_expressions.py b/packages/overture-schema-pyspark/tests/expressions/test_constraint_expressions.py index def5635f6..6d77605a5 100644 --- a/packages/overture-schema-pyspark/tests/expressions/test_constraint_expressions.py +++ b/packages/overture-schema-pyspark/tests/expressions/test_constraint_expressions.py @@ -14,6 +14,7 @@ check_enum, check_forbid_if, check_geometry_type, + check_integer, check_json_pointer, check_linear_range_bounds, check_linear_range_length, @@ -68,6 +69,61 @@ def test_except_literals_null_is_not_an_error(spark: SparkSession) -> None: assert _except_literals_error(spark, None) is None +def test_check_integer_integral_float_passes(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=2.0)], schema="val double") + result = df.select(check_integer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_integer_negative_integral_float_passes(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=-3.0)], schema="val double") + result = df.select(check_integer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_integer_fractional_float_violation(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=2.5)], schema="val double") + result = df.select(check_integer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + assert "integer" in result[0]["err"] + + +def test_check_integer_null_passthrough(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=None)], schema="val double") + result = df.select(check_integer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_integer_large_integral_double_passes(spark: SparkSession) -> None: + # Integral doubles beyond 2^63 are whole numbers Pydantic accepts + # ((1e30).is_integer() is True). A floor-based check would saturate the + # LongType cast and wrongly fire; the fractional-part check passes them. + df = spark.createDataFrame([Row(val=1e30)], schema="val double") + result = df.select(check_integer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is None + + +def test_check_integer_nan_violation(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=float("nan"))], schema="val double") + result = df.select(check_integer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + assert "integer" in result[0]["err"] + + +def test_check_integer_positive_infinity_violation(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=float("inf"))], schema="val double") + result = df.select(check_integer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + assert "integer" in result[0]["err"] + + +def test_check_integer_negative_infinity_violation(spark: SparkSession) -> None: + df = spark.createDataFrame([Row(val=float("-inf"))], schema="val double") + result = df.select(check_integer(F.col("val")).alias("err")).collect() + assert result[0]["err"] is not None + assert "integer" in result[0]["err"] + + def test_check_bounds_ge_le_valid(spark: SparkSession) -> None: df = spark.createDataFrame([Row(val=5)]) result = df.select(check_bounds(F.col("val"), ge=1, le=10).alias("err")).collect() diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_rail.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_rail.py index a59960996..d180f95a4 100644 --- a/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_rail.py +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_rail.py @@ -85,10 +85,9 @@ "recognized": ["as_permitted"], "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -590,10 +589,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -614,10 +612,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -640,10 +637,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -664,10 +660,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -690,10 +685,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -714,10 +708,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -742,7 +735,7 @@ { "dimension": "axle_count", "comparison": "greater_than", - "value": 0, + "value": 1.0, } ], }, @@ -753,6 +746,52 @@ expected_field="access_restrictions[].when.vehicle[].value_0", expected_check="required", ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value_0:bounds", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 1.0, + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", 0.0), + expected_field="access_restrictions[].when.vehicle[].value_0", + expected_check="bounds", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value_0:integer", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 1.0, + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", 1.5), + expected_field="access_restrictions[].when.vehicle[].value_0", + expected_check="integer", + ), Scenario( id="segment::access_restrictions[].when.vehicle[].value_1:required", scaffold={ @@ -778,7 +817,7 @@ expected_check="required", ), Scenario( - id="segment::access_restrictions[].when.vehicle[].value:bounds", + id="segment::access_restrictions[].when.vehicle[].value_1:bounds", scaffold={ "access_restrictions": [ { @@ -798,7 +837,7 @@ ] }, mutate=set_at_path("access_restrictions[].when.vehicle[].value", -1.0), - expected_field="access_restrictions[].when.vehicle[].value", + expected_field="access_restrictions[].when.vehicle[].value_1", expected_check="bounds", ), Scenario( @@ -1434,10 +1473,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -1465,10 +1503,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -1496,10 +1533,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -1527,10 +1563,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -1558,10 +1593,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_road.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_road.py index 2e58fcda5..0cfff5f42 100644 --- a/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_road.py +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_road.py @@ -85,10 +85,9 @@ "recognized": ["as_permitted"], "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -132,10 +131,9 @@ "recognized": ["as_permitted"], "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -157,10 +155,9 @@ "recognized": ["as_permitted"], "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -649,10 +646,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -673,10 +669,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -699,10 +694,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -723,10 +717,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -749,10 +742,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -773,10 +765,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -801,7 +792,7 @@ { "dimension": "axle_count", "comparison": "greater_than", - "value": 0, + "value": 1.0, } ], }, @@ -812,6 +803,52 @@ expected_field="access_restrictions[].when.vehicle[].value_0", expected_check="required", ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value_0:bounds", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 1.0, + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", 0.0), + expected_field="access_restrictions[].when.vehicle[].value_0", + expected_check="bounds", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value_0:integer", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 1.0, + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", 1.5), + expected_field="access_restrictions[].when.vehicle[].value_0", + expected_check="integer", + ), Scenario( id="segment::access_restrictions[].when.vehicle[].value_1:required", scaffold={ @@ -837,7 +874,7 @@ expected_check="required", ), Scenario( - id="segment::access_restrictions[].when.vehicle[].value:bounds", + id="segment::access_restrictions[].when.vehicle[].value_1:bounds", scaffold={ "access_restrictions": [ { @@ -857,7 +894,7 @@ ] }, mutate=set_at_path("access_restrictions[].when.vehicle[].value", -1.0), - expected_field="access_restrictions[].when.vehicle[].value", + expected_field="access_restrictions[].when.vehicle[].value_1", expected_check="bounds", ), Scenario( @@ -2170,10 +2207,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -2195,10 +2231,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -2222,10 +2257,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -2247,10 +2281,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -2274,10 +2307,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -2299,10 +2331,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -2328,7 +2359,7 @@ { "dimension": "axle_count", "comparison": "greater_than", - "value": 0, + "value": 1.0, } ], }, @@ -2339,6 +2370,54 @@ expected_field="prohibited_transitions[].when.vehicle[].value_0", expected_check="required", ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].value_0:bounds", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 1.0, + } + ], + }, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.vehicle[].value", 0.0), + expected_field="prohibited_transitions[].when.vehicle[].value_0", + expected_check="bounds", + ), + Scenario( + id="segment::prohibited_transitions[].when.vehicle[].value_0:integer", + scaffold={ + "prohibited_transitions": [ + { + "sequence": [{"connector_id": "a", "segment_id": "a"}], + "final_heading": "forward", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 1.0, + } + ], + }, + } + ] + }, + mutate=set_at_path("prohibited_transitions[].when.vehicle[].value", 1.5), + expected_field="prohibited_transitions[].when.vehicle[].value_0", + expected_check="integer", + ), Scenario( id="segment::prohibited_transitions[].when.vehicle[].value_1:required", scaffold={ @@ -2365,7 +2444,7 @@ expected_check="required", ), Scenario( - id="segment::prohibited_transitions[].when.vehicle[].value:bounds", + id="segment::prohibited_transitions[].when.vehicle[].value_1:bounds", scaffold={ "prohibited_transitions": [ { @@ -2386,7 +2465,7 @@ ] }, mutate=set_at_path("prohibited_transitions[].when.vehicle[].value", -1.0), - expected_field="prohibited_transitions[].when.vehicle[].value", + expected_field="prohibited_transitions[].when.vehicle[].value_1", expected_check="bounds", ), Scenario( @@ -2907,10 +2986,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -2931,10 +3009,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -2955,10 +3032,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -2979,10 +3055,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3003,10 +3078,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3027,10 +3101,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3053,7 +3126,7 @@ { "dimension": "axle_count", "comparison": "greater_than", - "value": 0, + "value": 1.0, } ], }, @@ -3064,6 +3137,52 @@ expected_field="speed_limits[].when.vehicle[].value_0", expected_check="required", ), + Scenario( + id="segment::speed_limits[].when.vehicle[].value_0:bounds", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 1.0, + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].value", 0.0), + expected_field="speed_limits[].when.vehicle[].value_0", + expected_check="bounds", + ), + Scenario( + id="segment::speed_limits[].when.vehicle[].value_0:integer", + scaffold={ + "speed_limits": [ + { + "max_speed": {"value": 1, "unit": "mph"}, + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 1.0, + } + ], + }, + } + ] + }, + mutate=set_at_path("speed_limits[].when.vehicle[].value", 1.5), + expected_field="speed_limits[].when.vehicle[].value_0", + expected_check="integer", + ), Scenario( id="segment::speed_limits[].when.vehicle[].value_1:required", scaffold={ @@ -3089,7 +3208,7 @@ expected_check="required", ), Scenario( - id="segment::speed_limits[].when.vehicle[].value:bounds", + id="segment::speed_limits[].when.vehicle[].value_1:bounds", scaffold={ "speed_limits": [ { @@ -3109,7 +3228,7 @@ ] }, mutate=set_at_path("speed_limits[].when.vehicle[].value", -1.0), - expected_field="speed_limits[].when.vehicle[].value", + expected_field="speed_limits[].when.vehicle[].value_1", expected_check="bounds", ), Scenario( @@ -3274,10 +3393,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3305,10 +3423,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3336,10 +3453,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3367,10 +3483,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3398,10 +3513,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3465,10 +3579,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3497,10 +3610,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3529,10 +3641,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3561,10 +3672,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3593,10 +3703,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3644,10 +3753,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3675,10 +3783,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3706,10 +3813,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3737,10 +3843,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -3768,10 +3873,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, diff --git a/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_water.py b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_water.py index 632f3757b..9e1a9e09f 100644 --- a/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_water.py +++ b/packages/overture-schema-pyspark/tests/generated/overture/schema/transportation/test_segment_water.py @@ -84,10 +84,9 @@ "recognized": ["as_permitted"], "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -587,10 +586,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -611,10 +609,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -637,10 +634,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -661,10 +657,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -687,10 +682,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -711,10 +705,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -739,7 +732,7 @@ { "dimension": "axle_count", "comparison": "greater_than", - "value": 0, + "value": 1.0, } ], }, @@ -750,6 +743,52 @@ expected_field="access_restrictions[].when.vehicle[].value_0", expected_check="required", ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value_0:bounds", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 1.0, + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", 0.0), + expected_field="access_restrictions[].when.vehicle[].value_0", + expected_check="bounds", + ), + Scenario( + id="segment::access_restrictions[].when.vehicle[].value_0:integer", + scaffold={ + "access_restrictions": [ + { + "access_type": "allowed", + "when": { + "heading": "forward", + "vehicle": [ + { + "dimension": "axle_count", + "comparison": "greater_than", + "value": 1.0, + } + ], + }, + } + ] + }, + mutate=set_at_path("access_restrictions[].when.vehicle[].value", 1.5), + expected_field="access_restrictions[].when.vehicle[].value_0", + expected_check="integer", + ), Scenario( id="segment::access_restrictions[].when.vehicle[].value_1:required", scaffold={ @@ -775,7 +814,7 @@ expected_check="required", ), Scenario( - id="segment::access_restrictions[].when.vehicle[].value:bounds", + id="segment::access_restrictions[].when.vehicle[].value_1:bounds", scaffold={ "access_restrictions": [ { @@ -795,7 +834,7 @@ ] }, mutate=set_at_path("access_restrictions[].when.vehicle[].value", -1.0), - expected_field="access_restrictions[].when.vehicle[].value", + expected_field="access_restrictions[].when.vehicle[].value_1", expected_check="bounds", ), Scenario( @@ -1354,10 +1393,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -1385,10 +1423,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -1416,10 +1453,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -1447,10 +1483,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, @@ -1478,10 +1513,9 @@ "heading": "forward", "vehicle": [ { - "dimension": "height", + "dimension": "axle_count", "comparison": "greater_than", - "value": 0.0, - "unit": "in", + "value": 1.0, } ], }, diff --git a/packages/overture-schema-system/src/overture/schema/system/field_constraint/__init__.py b/packages/overture-schema-system/src/overture/schema/system/field_constraint/__init__.py index aa031fcff..3ae2944c2 100644 --- a/packages/overture-schema-system/src/overture/schema/system/field_constraint/__init__.py +++ b/packages/overture-schema-system/src/overture/schema/system/field_constraint/__init__.py @@ -10,6 +10,10 @@ UniqueItemsConstraint, ) from .field_constraint import FieldConstraint +from .numeric import ( + IntegerConstraint, + NumericConstraint, +) from .string import ( CountryCodeAlpha2Constraint, HexColorConstraint, @@ -30,9 +34,11 @@ "CountryCodeAlpha2Constraint", "FieldConstraint", "HexColorConstraint", + "IntegerConstraint", "JsonPointerConstraint", "LanguageTagConstraint", "NoWhitespaceConstraint", + "NumericConstraint", "PatternConstraint", "PhoneNumberConstraint", "RegionCodeConstraint", diff --git a/packages/overture-schema-system/src/overture/schema/system/field_constraint/numeric.py b/packages/overture-schema-system/src/overture/schema/system/field_constraint/numeric.py new file mode 100644 index 000000000..1e230ea45 --- /dev/null +++ b/packages/overture-schema-system/src/overture/schema/system/field_constraint/numeric.py @@ -0,0 +1,79 @@ +""" +Constraints on fields with numeric values. +""" + +from typing import Any, NoReturn + +from pydantic import ( + GetCoreSchemaHandler, + GetJsonSchemaHandler, + ValidationError, + ValidationInfo, +) +from pydantic_core import InitErrorDetails, core_schema + +from .field_constraint import FieldConstraint + + +class NumericConstraint(FieldConstraint): + """Base class for numeric constraints. + + Wraps the field's own numeric schema (e.g. `float64` with any bounds) + with an after-validator, so a numeric constraint composes with the + bounds declared alongside it rather than replacing them. + """ + + def _raise_validation_error( + self, value: object, info: ValidationInfo, message: str + ) -> NoReturn: + context = info.context or {} + loc = context.get("loc_prefix", ()) + ("value",) + raise ValidationError.from_exception_data( + title=self.__class__.__name__, + line_errors=[ + InitErrorDetails( + type="value_error", + loc=loc, + input=value, + ctx={"error": message}, + ) + ], + ) + + def __get_pydantic_core_schema__( + self, source: type[Any], handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + numeric_schema = handler(source) + + def validate_numeric(value: Any, info: ValidationInfo) -> Any: + self.validate(value, info) + return value + + return core_schema.with_info_after_validator_function( + validate_numeric, numeric_schema + ) + + +class IntegerConstraint(NumericConstraint): + """Value must be integral (a whole number). + + Applies to floating-point fields whose values must not carry a + fractional part -- a count expressed as `float64` for physical-layout + reasons. Emits `multipleOf: 1` in JSON Schema, the conventional + encoding for "must be a whole number". + """ + + def validate(self, value: float | int | None, info: ValidationInfo) -> None: + if value is None: + return + if isinstance(value, float) and not value.is_integer(): + self._raise_validation_error( + value, info, f"must be an integer, got {value}" + ) + + def __get_pydantic_json_schema__( + self, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler + ) -> dict[str, Any]: + json_schema = handler(core_schema) + json_schema["multipleOf"] = 1 + return json_schema diff --git a/packages/overture-schema-system/tests/field_constraint/test_numeric_constraints.py b/packages/overture-schema-system/tests/field_constraint/test_numeric_constraints.py new file mode 100644 index 000000000..eceea0a60 --- /dev/null +++ b/packages/overture-schema-system/tests/field_constraint/test_numeric_constraints.py @@ -0,0 +1,64 @@ +from typing import Annotated + +import pytest +from pydantic import BaseModel, Field, ValidationError + +from overture.schema.system.field_constraint.numeric import IntegerConstraint + + +class _IntModel(BaseModel): + value: Annotated[float, IntegerConstraint()] + + +# Bound before the constraint so pydantic attaches `ge` to the numeric schema +# (emitting standard JSON Schema `minimum`) rather than to the wrapping +# after-validator (which would leak a raw `ge`). +class _BoundedIntModel(BaseModel): + value: Annotated[float, Field(ge=1), IntegerConstraint()] + + +class TestIntegerConstraintValidation: + @pytest.mark.parametrize("value", [0.0, 1.0, 2.0, -3.0, 255.0]) + def test_integral_float_accepted(self, value: float) -> None: + model = _IntModel(value=value) + assert model.value == value + + def test_integer_input_accepted(self) -> None: + model = _IntModel(value=2) + assert model.value == 2.0 + + @pytest.mark.parametrize("value", [2.5, 0.1, -1.5, 3.14]) + def test_non_integral_float_rejected(self, value: float) -> None: + with pytest.raises(ValidationError) as exc_info: + _IntModel(value=value) + assert "integer" in str(exc_info.value).lower() + + def test_bounds_still_enforced_alongside_integer(self) -> None: + assert _BoundedIntModel(value=1.0).value == 1.0 + with pytest.raises(ValidationError): + _BoundedIntModel(value=0.0) # integral but violates ge=1 + + def test_non_integral_rejected_with_bounds(self) -> None: + with pytest.raises(ValidationError): + _BoundedIntModel(value=1.5) # satisfies ge=1 but not integral + + +class TestIntegerConstraintJsonSchema: + def test_multiple_of_one_emitted(self) -> None: + schema = _IntModel.model_json_schema() + value_schema = schema["properties"]["value"] + assert value_schema["multipleOf"] == 1 + assert value_schema["type"] == "number" + + def test_multiple_of_coexists_with_minimum(self) -> None: + schema = _BoundedIntModel.model_json_schema() + value_schema = schema["properties"]["value"] + assert value_schema["multipleOf"] == 1 + assert value_schema["minimum"] == 1 + + +class TestIntegerConstraintEquality: + def test_instances_are_equal_and_hash_equal(self) -> None: + assert IntegerConstraint() == IntegerConstraint() + assert hash(IntegerConstraint()) == hash(IntegerConstraint()) + assert len({IntegerConstraint(), IntegerConstraint()}) == 1 diff --git a/packages/overture-schema-transportation-theme/tests/segment_baseline_schema.json b/packages/overture-schema-transportation-theme/tests/segment_baseline_schema.json index d4e31ffab..45d6ce3b9 100644 --- a/packages/overture-schema-transportation-theme/tests/segment_baseline_schema.json +++ b/packages/overture-schema-transportation-theme/tests/segment_baseline_schema.json @@ -1394,10 +1394,10 @@ }, "value": { "description": "Number of axles on the vehicle", - "maximum": 255, - "minimum": 0, + "minimum": 1, + "multipleOf": 1, "title": "Value", - "type": "integer" + "type": "number" } }, "required": [