From e90e37468917bb12cdf97393ff1d355f2a39b24a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 14 Jun 2026 10:14:38 +0000 Subject: [PATCH 1/7] feat(schema-docs): implement US5 generated schema reference + ERD (T024-T027) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generate the schema reference and Mermaid ERD from the enriched XSD so docs can't drift (FR-020/021/022, SC-008/SC-009). - schema_docs.py: parse the enriched XSD and render a generated Markdown reference — a Mermaid erDiagram of entities + relationships (with exact cardinalities), per-complex-type field tables carrying the xs:documentation prose (inheritance flattened and marked), and a banded-numeric-type catalogue with range facets. Deterministic/idempotent. - cli.py + Makefile: wire `gen-schema-docs` subcommand and `make gen-schema-docs`. - docs/reference/schema/index.md: the generated artifact (committed; regenerated by CI). - docs: delete the hand-drawn schema-erd.md stand-in and repoint all inbound links to the generated page; integrate it via a single static nav entry (no extra plugin needed). - tests/integration/test_schema_docs.py: entities present, xs:documentation prose carried, erDiagram emitted, inheritance flattened, idempotent, and committed page == fresh regen. - tasks.md/commands.md/onboarding.md: mark US5 done; remaining work is US4 (bundle + drift). --- Makefile | 4 +- docs/concepts/pipeline-data-flow.md | 9 +- docs/glossary.md | 2 +- docs/how-to/build-the-docs-site.md | 10 +- docs/index.md | 2 +- docs/onboarding.md | 7 +- docs/reference/commands.md | 6 +- docs/reference/index.md | 11 +- docs/reference/schema-erd.md | 69 ----- docs/reference/schema/index.md | 186 ++++++++++++ mkdocs.yml | 2 +- specs/001-codespace-xml-scaffold/tasks.md | 25 +- src/acoustic_dataset/cli.py | 16 ++ src/acoustic_dataset/schema_docs.py | 328 ++++++++++++++++++++++ tests/integration/test_schema_docs.py | 69 +++++ 15 files changed, 640 insertions(+), 106 deletions(-) delete mode 100644 docs/reference/schema-erd.md create mode 100644 docs/reference/schema/index.md create mode 100644 src/acoustic_dataset/schema_docs.py create mode 100644 tests/integration/test_schema_docs.py diff --git a/Makefile b/Makefile index 3f69bb5..a0a3436 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,8 @@ docs-serve: ## Live-preview the HTML docs at http://localhost:8000 generate: ## Regenerate typed models from schema/*.xsd via xsdata. python -m acoustic_dataset.cli generate -gen-schema-docs: ## Generate schema reference pages + Mermaid ERD from the enriched XSD. (Phase 1 task) - @echo "Not yet implemented — tracked in specs/001-codespace-xml-scaffold/tasks.md"; exit 1 +gen-schema-docs: ## Generate schema reference pages + Mermaid ERD from the enriched XSD. + python -m acoustic_dataset.cli gen-schema-docs pipeline: ## End-to-end: map example input -> objects -> XML -> validate -> round-trip. python -m acoustic_dataset.cli pipeline diff --git a/docs/concepts/pipeline-data-flow.md b/docs/concepts/pipeline-data-flow.md index dca5db2..f6c2ce9 100644 --- a/docs/concepts/pipeline-data-flow.md +++ b/docs/concepts/pipeline-data-flow.md @@ -28,10 +28,11 @@ intermediate (no CSV, no pickle) is needed to get testability; that whole chain ## The entities as an ER diagram -This is the kind of **Mermaid ERD** the docs render — here drawn by hand for the *placeholder* -domain. Once the real enriched XSD lands, `make gen-schema-docs` produces the equivalent -diagram **automatically from the schema** (see [the example schema ERD](../reference/schema-erd.md) -and [ADR 0009](../decisions/0009-mkdocs-material-mermaid-html-docs.md)). +This is the kind of **Mermaid ERD** the docs render — here drawn by hand for the data-flow story +(it deliberately includes pipeline entities like the golden and reference files). The +**[schema reference](../reference/schema/index.md)** ERD, by contrast, is produced +**automatically from the schema** by `make gen-schema-docs` +([ADR 0009](../decisions/0009-mkdocs-material-mermaid-html-docs.md)). ```mermaid erDiagram diff --git a/docs/glossary.md b/docs/glossary.md index d98c084..3461899 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -46,7 +46,7 @@ comparison, not by validation alone. **ERD (Entity-Relationship Diagram)** — a diagram of entities and how they relate. We render these with **Mermaid** and generate them from the enriched XSD so they cannot drift -from the contract. See the [example ERD](reference/schema-erd.md). +from the contract. See the [generated schema reference](reference/schema/index.md). ## Tooling diff --git a/docs/how-to/build-the-docs-site.md b/docs/how-to/build-the-docs-site.md index a647a43..757f65c 100644 --- a/docs/how-to/build-the-docs-site.md +++ b/docs/how-to/build-the-docs-site.md @@ -45,14 +45,14 @@ erDiagram ```` See the [pipeline ERD](../concepts/pipeline-data-flow.md) and the -[example schema ERD](../reference/schema-erd.md) for working examples. +[generated schema ERD](../reference/schema/index.md) for working examples. ## Schema reference & ERD are *generated* -The pages under `reference/` that describe the schema are produced from the **enriched XSD** -by `make gen-schema-docs` (a Phase 1 task), not hand-written — so they can't drift from the -contract. Until that step is implemented, the [example ERD](../reference/schema-erd.md) is a -hand-drawn stand-in showing the intended output. +The [schema reference](../reference/schema/index.md) — the entity tables and the Mermaid ERD — +is produced from the **enriched XSD** by `make gen-schema-docs`, not hand-written, so it can't +drift from the contract. Regenerate it after any schema change; the CI drift gate fails if the +committed page is stale. ## Linking rules (to keep `--strict` happy) diff --git a/docs/index.md b/docs/index.md index c61d235..dbd2e12 100644 --- a/docs/index.md +++ b/docs/index.md @@ -38,7 +38,7 @@ human semantic gate (is the science right?). See [Add a decision record](how-to/add-a-decision-record.md). - New understanding lands as a **concept** page; new recipes as **how-to** guides. - The **schema reference and ERD** are *generated from the enriched XSD* — see the - [example ERD](reference/schema-erd.md) — so the docs can never drift from the contract. + [generated schema reference](reference/schema/index.md) — so the docs can never drift from the contract. !!! note "Status" Provisional scaffold. The environment (Codespace, tests, docs site) is real and green diff --git a/docs/onboarding.md b/docs/onboarding.md index 2dafc60..f739a55 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -31,6 +31,7 @@ before the same `make verify` / `make pipeline`. Both paths reach the *same* gre | Example calculation input (placeholder) | `examples/calculation_input.json` | | Tests (unit / integration / golden) | `tests/` | | The plan & design artifacts | `specs/001-codespace-xml-scaffold/` (`spec.md`, `plan.md`, `tasks.md`) | +| The generated schema reference + ERD | [reference/schema](reference/schema/index.md) (run `make gen-schema-docs`) | | Why each choice was made | [Decision records](decisions/index.md) | ## Build the mental model @@ -42,6 +43,6 @@ Read these, in order — they're short: 3. [Pipeline data flow](concepts/pipeline-data-flow.md) — how input becomes validated XML. !!! tip "What's done, what's next" - The environment, the end-to-end pipeline, and the migration-safety `compare` are in - place. Remaining pipeline work (generated schema docs/ERD and the distribution bundle) - is tracked in `specs/001-codespace-xml-scaffold/tasks.md`. + The environment, the end-to-end pipeline, the migration-safety `compare`, and the + generated schema reference/ERD are in place. The remaining pipeline work (the distribution + bundle + CI drift gate) is tracked in `specs/001-codespace-xml-scaffold/tasks.md`. diff --git a/docs/reference/commands.md b/docs/reference/commands.md index dfe2d37..9a1b69e 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -13,13 +13,13 @@ | `make docs` | Build static HTML site (`mkdocs build --strict`) → `site/` | ✅ | | `make docs-serve` | Live-preview docs at | ✅ | | `make generate` | Regenerate models from `schema/*.xsd` via `xsdata` | ✅ | -| `make gen-schema-docs` | Generate schema reference + Mermaid ERD from the enriched XSD | ⏳ Phase 1 task | +| `make gen-schema-docs` | Generate schema reference + Mermaid ERD from the enriched XSD | ✅ | | `make pipeline` | End-to-end: map → serialise → validate → round-trip | ✅ | | `make compare` | Migration-safety diff vs a known-good reference | ✅ | | `make bundle` | Distribution bundle (data + schema + models) | ⏳ Phase 1 task | -⏳ targets currently print a pointer to `specs/001-codespace-xml-scaffold/tasks.md` and exit -non-zero. +The remaining ⏳ target (`make bundle`, User Story 4) currently prints a pointer to +`specs/001-codespace-xml-scaffold/tasks.md` and exits non-zero. ## CLI subcommands diff --git a/docs/reference/index.md b/docs/reference/index.md index 8cd6dbd..3449b43 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -5,7 +5,7 @@ | Page | What it gives you | |---|---| | [Commands](commands.md) | Every `make` target and CLI command, with exit-code meaning | -| [Schema ERD (example)](schema-erd.md) | The Mermaid ER diagram the docs generate from the enriched XSD | +| [Schema reference](schema/index.md) | Generated entity tables + Mermaid ERD, produced from the enriched XSD | ## Authoritative specifications @@ -17,8 +17,9 @@ the docs site, under version control): - `specs/001-codespace-xml-scaffold/data-model.md` — entities and field rules. - `specs/001-codespace-xml-scaffold/contracts/` — the CLI and pipeline contracts. -## Generated reference (coming with the real schema) +## Generated reference -When the enriched XSD lands, `make gen-schema-docs` adds **generated** pages here — one per -schema type, carrying the `xs:documentation` prose, plus the Mermaid ERD — produced *from the -schema* so they cannot drift ([ADR 0009](../decisions/0009-mkdocs-material-mermaid-html-docs.md)). +`make gen-schema-docs` (re)generates the [schema reference](schema/index.md) from the enriched +XSD — the entity tables, every field's `xs:documentation` prose, and the Mermaid ERD — produced +*from the schema* so they cannot drift ([ADR 0009](../decisions/0009-mkdocs-material-mermaid-html-docs.md)). +The committed page is regenerated by CI; a stale copy fails the drift gate. diff --git a/docs/reference/schema-erd.md b/docs/reference/schema-erd.md deleted file mode 100644 index e32f57e..0000000 --- a/docs/reference/schema-erd.md +++ /dev/null @@ -1,69 +0,0 @@ -# Schema ERD (example) - -> **Reference** — an example of the **generated** Mermaid ER diagram. - -!!! info "This is a hand-drawn stand-in" - Once the real enriched XSD is in place, `make gen-schema-docs` produces this page - **automatically from the schema** — entities, fields, relationships, and the - `xs:documentation` prose all read from the XSD so the diagram can't drift from the contract - ([ADR 0009](../decisions/0009-mkdocs-material-mermaid-html-docs.md)). The diagram below - shows the intended shape for the *placeholder* schema. - -## Entities and relationships - -```mermaid -erDiagram - PLATFORM ||--|| CHARACTERISTICS : describes - PLATFORM ||--o{ RADIATED_BAND : "radiates across (10 bands)" - RADIATED_BAND ||--o{ SECTOR : "all round (12 x 30 deg)" - PLATFORM ||--|| SENSOR_SUITE : carries - SENSOR_SUITE ||--|| ACTIVE_SONAR : "1 active" - SENSOR_SUITE ||--o{ PASSIVE_SONAR : "2 passive" - - PLATFORM { - string name "Human-readable name of the platform" - date generated_on "When the document was produced" - string schema_version "Version of the contract used" - } - CHARACTERISTICS { - decimal draft "Draft below the waterline, in metres" - decimal length "Overall length, in metres" - decimal weight "Displacement, in tonnes" - int year_introduced "Year the class entered service" - } - RADIATED_BAND { - int index "1-based ordinal of the band" - decimal centre_frequency "Centre frequency of the band, in hertz" - } - SECTOR { - decimal bearing "Centre bearing of the sector, in degrees [0, 360)" - decimal level "Radiated noise level in this sector, in decibels" - } - ACTIVE_SONAR { - string name "Model name of the sonar" - decimal source_level "Transmit source level, in decibels" - decimal max_range "Maximum echo detection range, in metres" - } - PASSIVE_SONAR { - string name "Model name of the sonar" - decimal array_gain "Array gain, in decibels" - decimal bearing_accuracy "1-sigma bearing accuracy, in degrees" - } -``` - -## How to read it - -- `||--o{` means "one to zero-or-many" — one `PLATFORM` radiates across many - `RADIATED_BAND`s, each holding many `SECTOR`s. -- `||--||` means "one to one". -- The text after each field (e.g. *"Lower edge of the band"*) is exactly what the **enriched - XSD** will carry in `xs:annotation/xs:documentation` — the same prose that becomes the - generated data-class docstrings. One source, many outputs - ([Schema as the contract](../concepts/schema-as-contract.md)). - -## Why generate it (rather than draw it) - -A hand-drawn diagram is a second source of truth that rots. Generating the ERD from the schema -means the picture and the contract are *the same artefact viewed two ways* — change the schema, -regenerate, and the diagram is correct by construction -([ADR 0008](../decisions/0008-generated-models-no-drift.md)). diff --git a/docs/reference/schema/index.md b/docs/reference/schema/index.md new file mode 100644 index 0000000..1d38537 --- /dev/null +++ b/docs/reference/schema/index.md @@ -0,0 +1,186 @@ + + +# Schema reference + +> **Reference (generated)** — produced from `schema/acoustic_dataset.xsd` (version `0.2.0-placeholder`) by `make gen-schema-docs`. Every entity, field, range and definition below is read from the XSD's `xs:annotation/xs:documentation`, so this page cannot drift from the contract. + +Placeholder Platform schema. A document describes one platform: its physical characteristics, its directional radiated-noise signature across ten frequency bands, and the sonar sensors it carries. The banded numeric types below carry real XSD facets (ranges) and the radiated-noise structure carries enforced cardinalities (exactly ten bands, exactly twelve 30-degree sectors per band) so the validation gate has something meaningful to enforce. + +## Entity-relationship diagram + +```mermaid +erDiagram + DIRECTIONAL ||--|{ SECTOR : "Sector (12)" + RADIATED_BAND ||--|| DIRECTIONAL : "Directional" + RADIATED_NOISE ||--|{ RADIATED_BAND : "Band (10)" + SENSOR_SUITE ||--|| ACTIVE_SONAR : "Active" + SENSOR_SUITE ||--|{ PASSIVE_SONAR : "Passive (2)" + PLATFORM_TYPE ||--|| PLATFORM_CHARACTERISTICS : "Characteristics" + PLATFORM_TYPE ||--|| RADIATED_NOISE : "RadiatedNoise" + PLATFORM_TYPE ||--|| SENSOR_SUITE : "Sensors" + PLATFORM_CHARACTERISTICS { + Metres Draft "Draft (depth of the lowest point below the waterline), in metres." + Metres Length "Overall length of the platform, in metres." + Tonnes Weight "Displacement (weight) of the platform, in tonnes." + Year YearIntroduced "Calendar year the platform class entered service." + } + SECTOR { + Bearing Bearing "Centre bearing of the sector, in degrees [0, 360)." + Decibels Level "Radiated noise level in this sector, in decibels." + } + RADIATED_BAND { + Hertz CentreFrequency "Centre frequency of the band, in hertz." + positiveInteger index "1-based ordinal of the band within the signature." + } + ACTIVE_SONAR { + string Name "Model name of the sonar." + string Manufacturer "Manufacturer of the sonar." + Hertz OperatingFrequency "Nominal operating (centre) frequency of the sonar, in hertz." + Decibels SourceLevel "Transmit source level, in decibels." + Degrees Beamwidth "Transmit/receive beamwidth, in degrees." + Seconds PulseLength "Transmit pulse length, in seconds." + Metres MaxRange "Maximum echo detection range, in metres." + } + PASSIVE_SONAR { + string Name "Model name of the sonar." + string Manufacturer "Manufacturer of the sonar." + Hertz OperatingFrequency "Nominal operating (centre) frequency of the sonar, in hertz." + Decibels ArrayGain "Array gain of the receiving array, in decibels." + Decibels DetectionThreshold "Signal-to-noise ratio required for detection, in decibels." + Degrees BearingAccuracy "1-sigma bearing accuracy of the sonar, in degrees." + } + PLATFORM_TYPE { + string SchemaVersion "Version of the schema this document targets." + string Name "Human-readable name of the platform." + dateTime GeneratedUtc "UTC timestamp identifying when the document was produced." + } +``` + +Legend: `||--||` one-to-one, `||--|{` one-to-(one-or-many), `||--o{` one-to-(zero-or-many). The number in each label is the exact cardinality the schema enforces. Sonar sub-types show their inherited fields inline. + +## Entities + +### PlatformCharacteristics + +The physical characteristics of the platform. + +| Field | Type | Cardinality | Definition | +|---|---|---|---| +| `Draft` | [`Metres`](#banded-numeric-types) | 1 | Draft (depth of the lowest point below the waterline), in metres. | +| `Length` | [`Metres`](#banded-numeric-types) | 1 | Overall length of the platform, in metres. | +| `Weight` | [`Tonnes`](#banded-numeric-types) | 1 | Displacement (weight) of the platform, in tonnes. | +| `YearIntroduced` | [`Year`](#banded-numeric-types) | 1 | Calendar year the platform class entered service. | + +### Sector + +The radiated noise level in one 30-degree bearing sector of a band. + +| Field | Type | Cardinality | Definition | +|---|---|---|---| +| `Bearing` | [`Bearing`](#banded-numeric-types) | 1 | Centre bearing of the sector, in degrees [0, 360). | +| `Level` | [`Decibels`](#banded-numeric-types) | 1 | Radiated noise level in this sector, in decibels. | + +### Directional + +The all-round radiated noise for one band: exactly twelve sectors at 30-degree intervals, in ascending bearing order (0, 30, ..., 330). + +| Field | Type | Cardinality | Definition | +|---|---|---|---| +| `Sector` | [`Sector`](#sector) | 12 | One 30-degree bearing sector. | + +### RadiatedBand + +The directional radiated noise for one frequency band. + +| Field | Type | Cardinality | Definition | +|---|---|---|---| +| `CentreFrequency` | [`Hertz`](#banded-numeric-types) | 1 | Centre frequency of the band, in hertz. | +| `Directional` | [`Directional`](#directional) | 1 | The twelve 30-degree directional noise sectors for this band. | +| `index` | `xs:positiveInteger` | 1 (attribute) | 1-based ordinal of the band within the signature. | + +### RadiatedNoise + +The platform's radiated-noise signature: exactly ten frequency bands, in ascending index order. + +| Field | Type | Cardinality | Definition | +|---|---|---|---| +| `Band` | [`RadiatedBand`](#radiatedband) | 10 | One frequency band of the radiated-noise signature. | + +### Sonar + +Fields common to every sonar: identity plus a nominal operating frequency. + +| Field | Type | Cardinality | Definition | +|---|---|---|---| +| `Name` | `xs:string` | 1 | Model name of the sonar. | +| `Manufacturer` | `xs:string` | 1 | Manufacturer of the sonar. | +| `OperatingFrequency` | [`Hertz`](#banded-numeric-types) | 1 | Nominal operating (centre) frequency of the sonar, in hertz. | + +### ActiveSonar + +An active sonar: it transmits, so it carries a source level and beam/pulse figures, and a derived maximum (echo) detection range. + +Extends [`Sonar`](#sonar); inherited fields are marked below. + +| Field | Type | Cardinality | Definition | +|---|---|---|---| +| `Name` *(from Sonar)* | `xs:string` | 1 | Model name of the sonar. | +| `Manufacturer` *(from Sonar)* | `xs:string` | 1 | Manufacturer of the sonar. | +| `OperatingFrequency` *(from Sonar)* | [`Hertz`](#banded-numeric-types) | 1 | Nominal operating (centre) frequency of the sonar, in hertz. | +| `SourceLevel` | [`Decibels`](#banded-numeric-types) | 1 | Transmit source level, in decibels. | +| `Beamwidth` | [`Degrees`](#banded-numeric-types) | 1 | Transmit/receive beamwidth, in degrees. | +| `PulseLength` | [`Seconds`](#banded-numeric-types) | 1 | Transmit pulse length, in seconds. | +| `MaxRange` | [`Metres`](#banded-numeric-types) | 1 | Maximum echo detection range, in metres. | + +### PassiveSonar + +A passive sonar: it only listens, so it carries array gain, a detection threshold and a bearing accuracy rather than a transmit source level. + +Extends [`Sonar`](#sonar); inherited fields are marked below. + +| Field | Type | Cardinality | Definition | +|---|---|---|---| +| `Name` *(from Sonar)* | `xs:string` | 1 | Model name of the sonar. | +| `Manufacturer` *(from Sonar)* | `xs:string` | 1 | Manufacturer of the sonar. | +| `OperatingFrequency` *(from Sonar)* | [`Hertz`](#banded-numeric-types) | 1 | Nominal operating (centre) frequency of the sonar, in hertz. | +| `ArrayGain` | [`Decibels`](#banded-numeric-types) | 1 | Array gain of the receiving array, in decibels. | +| `DetectionThreshold` | [`Decibels`](#banded-numeric-types) | 1 | Signal-to-noise ratio required for detection, in decibels. | +| `BearingAccuracy` | [`Degrees`](#banded-numeric-types) | 1 | 1-sigma bearing accuracy of the sonar, in degrees. | + +### SensorSuite + +The sonar fit carried by the platform: one active sonar and two passive sonars. + +| Field | Type | Cardinality | Definition | +|---|---|---|---| +| `Active` | [`ActiveSonar`](#activesonar) | 1 | The platform's single active sonar. | +| `Passive` | [`PassiveSonar`](#passivesonar) | 2 | The platform's two passive sonars. | + +### PlatformType (root element `Platform`) + +A single platform: titled, timestamped reference data in three parts — physical characteristics, directional radiated noise, and the sonar fit. + +| Field | Type | Cardinality | Definition | +|---|---|---|---| +| `SchemaVersion` | `xs:string` | 1 | Version of the schema this document targets. | +| `Name` | `xs:string` | 1 | Human-readable name of the platform. | +| `GeneratedUtc` | `xs:dateTime` | 1 | UTC timestamp identifying when the document was produced. | +| `Characteristics` | [`PlatformCharacteristics`](#platformcharacteristics) | 1 | The platform's physical characteristics. | +| `RadiatedNoise` | [`RadiatedNoise`](#radiatednoise) | 1 | The platform's directional radiated-noise signature. | +| `Sensors` | [`SensorSuite`](#sensorsuite) | 1 | The platform's sonar fit. | + +## Banded numeric types + +The numeric primitives below carry real XSD range facets, so an out-of-band value fails the validation gate. + +| Type | Base | Range | Definition | +|---|---|---|---| +| `Decibels` | `xs:decimal` | ≥ -200, ≤ 300 | A level expressed in decibels (dB), bounded to a sane sonar range. | +| `Metres` | `xs:decimal` | ≥ 0 | A non-negative distance in metres. | +| `Hertz` | `xs:decimal` | ≥ 0 | A non-negative frequency in hertz (Hz). | +| `Tonnes` | `xs:decimal` | ≥ 0 | A non-negative mass/displacement in tonnes (1000 kg). | +| `Seconds` | `xs:decimal` | ≥ 0 | A non-negative duration in seconds. | +| `Degrees` | `xs:decimal` | ≥ 0, ≤ 360 | An angle in degrees, in the closed interval [0, 360] (e.g. a beamwidth). | +| `Bearing` | `xs:decimal` | ≥ 0, < 360 | A compass bearing in degrees, in the half-open interval [0, 360): 0 is dead ahead, increasing clockwise. Radiated-noise sectors are spaced 30 degrees apart. | +| `Year` | `xs:integer` | ≥ 1900, ≤ 2100 | A Gregorian calendar year, constrained to a sane modern range [1900, 2100]. | + diff --git a/mkdocs.yml b/mkdocs.yml index c87b251..2b9de69 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -69,5 +69,5 @@ nav: - Reference: - Overview: reference/index.md - Commands: reference/commands.md - - Schema ERD (example): reference/schema-erd.md + - Schema reference (generated): reference/schema/index.md - Glossary: glossary.md diff --git a/specs/001-codespace-xml-scaffold/tasks.md b/specs/001-codespace-xml-scaffold/tasks.md index 85ea759..9585e17 100644 --- a/specs/001-codespace-xml-scaffold/tasks.md +++ b/specs/001-codespace-xml-scaffold/tasks.md @@ -17,11 +17,12 @@ round-trip gate (FR-009, FR-010, FR-014), so test tasks are first-class here. > (Foundational) and **User Story 2 — the schema-driven pipeline MVP** are now implemented: > enriched placeholder XSD, xsdata generation (3.9-safe, idempotent), the single mapping, the > two structural gates, `make generate` / `make pipeline`, and the golden-file tests. -> **User Story 3 — the migration-safety `compare`** is now implemented too: a canonicalising -> comparison (prefix/attribute/whitespace/comment-insensitive), `make compare`, a shipped -> known-good reference fixture, and its tests. The **remaining work is US5 (generated schema -> docs/ERD) and US4 (bundle + drift gate)**; the `bundle` CLI subcommand exists but exits -> non-zero as not-yet-implemented. +> **User Story 3 — the migration-safety `compare`** and **User Story 5 — the generated schema +> reference & ERD** are now implemented too: a canonicalising comparison (`make compare`) with a +> shipped reference fixture, and an XSD-driven Markdown reference + Mermaid `erDiagram` +> (`make gen-schema-docs`) wired into the docs nav, both with tests. The **remaining work is US4 +> (bundle + drift gate)**; the `bundle` CLI subcommand exists but exits non-zero as +> not-yet-implemented. ## Format: `[ID] [P?] [Story] Description` @@ -101,18 +102,18 @@ round-trip gate (FR-009, FR-010, FR-014), so test tasks are first-class here. --- -## Phase 6: User Story 5 - Generated schema reference & ERD (Priority: P2) [FR-020/021/022] +## Phase 6: User Story 5 - Generated schema reference & ERD (Priority: P2) [FR-020/021/022] — ✅ COMPLETE **Goal**: Generate the HTML schema reference + a Mermaid **ERD** *from the enriched XSD*, so docs can't drift. **Independent Test**: `make gen-schema-docs` then `make docs` renders generated reference pages and an ERD matching the schema. -- [ ] T024 [US5] Implement the schema-docs generator (walk the enriched XSD/models → per-type Markdown reference carrying `xs:documentation` + a Mermaid `erDiagram`) into `docs/reference/schema/` in `src/acoustic_dataset/schema_docs.py` (depends on T010) -- [ ] T025 [US5] Wire `gen-schema-docs` subcommand + `make gen-schema-docs` in `src/acoustic_dataset/cli.py` (depends on T024) -- [ ] T026 [US5] Integrate generated pages into the MkDocs nav (e.g. `mkdocs-gen-files` + `mkdocs-literate-nav` or `awesome-pages`) and replace the hand-drawn stand-in note in `docs/reference/schema-erd.md`; update `mkdocs.yml`/`pyproject.toml` docs extra -- [ ] T027 [P] [US5] Test generator output contains entities, the `xs:documentation` prose, and an `erDiagram` block in `tests/integration/test_schema_docs.py` +- [X] T024 [US5] Implement the schema-docs generator (walk the enriched XSD/models → per-type Markdown reference carrying `xs:documentation` + a Mermaid `erDiagram`) into `docs/reference/schema/` in `src/acoustic_dataset/schema_docs.py` (depends on T010) +- [X] T025 [US5] Wire `gen-schema-docs` subcommand + `make gen-schema-docs` in `src/acoustic_dataset/cli.py` (depends on T024) +- [X] T026 [US5] Integrate the generated page into the MkDocs nav (single static `reference/schema/index.md` entry — no extra plugin needed) and replace the hand-drawn stand-in `docs/reference/schema-erd.md` (deleted; all inbound links repointed) +- [X] T027 [P] [US5] Test generator output contains entities, the `xs:documentation` prose, and an `erDiagram` block in `tests/integration/test_schema_docs.py` -**Checkpoint**: HTML site shows a schema ERD generated from the XSD (SC-008, SC-009). +**Checkpoint**: HTML site shows a schema ERD generated from the XSD (SC-008, SC-009). ✅ --- @@ -194,4 +195,4 @@ placeholder once it arrives. - `[P]` = different files, no dependencies. `[Story]` ties a task to a user story for traceability. - Generated artifacts (`src/acoustic_dataset/models/`, `docs/reference/schema/`) are **never** hand-edited — regenerate (ADR 0008). - Commit after each task or logical group; keep `make verify` green. -- Total: 34 tasks — 23 already complete (`[X]`), 11 remaining (the pipeline). +- Total: 34 tasks — 27 already complete (`[X]`), 7 remaining (the pipeline). diff --git a/src/acoustic_dataset/cli.py b/src/acoustic_dataset/cli.py index 7d06dae..2878d72 100644 --- a/src/acoustic_dataset/cli.py +++ b/src/acoustic_dataset/cli.py @@ -78,6 +78,15 @@ def cmd_validate(args: argparse.Namespace) -> int: return 1 +def cmd_gen_schema_docs(args: argparse.Namespace) -> int: + """Generate the schema reference + Mermaid ERD from the enriched XSD.""" + from acoustic_dataset import schema_docs + + out_file = schema_docs.generate(args.schema, args.out) + print(f"schema docs ok: generated {out_file} from {args.schema}") + return 0 + + def cmd_compare(args: argparse.Namespace) -> int: """Migration-safety diff: catch schema-valid-but-different output vs a reference.""" from lxml import etree @@ -138,6 +147,13 @@ def build_parser() -> argparse.ArgumentParser: p_val.add_argument("--schema", type=Path, default=DEFAULT_SCHEMA) p_val.set_defaults(func=cmd_validate) + p_doc = sub.add_parser( + "gen-schema-docs", help="Generate schema reference + Mermaid ERD from the XSD (US5)." + ) + p_doc.add_argument("--schema", type=Path, default=DEFAULT_SCHEMA) + p_doc.add_argument("--out", type=Path, default=_REPO_ROOT / "docs" / "reference" / "schema") + p_doc.set_defaults(func=cmd_gen_schema_docs) + p_cmp = sub.add_parser("compare", help="Migration-safety diff vs a reference (US3).") p_cmp.add_argument("generated", type=Path, help="The freshly generated XML to check.") p_cmp.add_argument( diff --git a/src/acoustic_dataset/schema_docs.py b/src/acoustic_dataset/schema_docs.py new file mode 100644 index 0000000..c9b429a --- /dev/null +++ b/src/acoustic_dataset/schema_docs.py @@ -0,0 +1,328 @@ +"""Generate the schema reference + Mermaid ERD from the enriched XSD (US5, FR-020/021/022). + +The enriched XSD is the single source of truth. This walks it and emits a Markdown reference — +a Mermaid ``erDiagram`` of the entities and their relationships, a per-complex-type field +reference carrying the ``xs:documentation`` prose, and a catalogue of the banded numeric types — +into ``docs/reference/schema/``. The page is a *generated artifact*, never hand-edited, so the +reference and the diagram cannot drift from the contract (ADR 0008/0009); the CI drift gate +regenerates it and fails on any diff. +""" + +from __future__ import annotations + +import dataclasses +import re +from dataclasses import dataclass +from pathlib import Path + +from lxml import etree + +_XS = "http://www.w3.org/2001/XMLSchema" +_NS = {"xs": _XS} + +_FACET_ORDER = ("minInclusive", "minExclusive", "maxInclusive", "maxExclusive") +_FACET_OP = { + "minInclusive": "≥", + "minExclusive": ">", + "maxInclusive": "≤", + "maxExclusive": "<", +} + + +@dataclass +class SchemaField: + """One element or attribute of a complex type.""" + + name: str + type_ref: str # raw @type, e.g. "Metres" or "xs:string" + is_attribute: bool + min_occurs: int + max_occurs: str # an int as a string, or "unbounded" + required: bool + doc: str + inherited_from: str = "" + + @property + def type_local(self) -> str: + return self.type_ref.split(":")[-1] if self.type_ref else "" + + +@dataclass +class ComplexType: + name: str + doc: str + base: str # base complex-type name, or "" + fields: list[SchemaField] + + +@dataclass +class SimpleType: + name: str + base: str + facets: dict[str, str] + doc: str + + +@dataclass +class SchemaModel: + version: str + doc: str + simple_types: list[SimpleType] + complex_types: list[ComplexType] + root_element: str + root_type: str + + +# --------------------------------------------------------------------------- parsing + + +def _collapse(text: str) -> str: + return re.sub(r"\s+", " ", text).strip() + + +def _doc(node: etree._Element) -> str: + doc = node.find("xs:annotation/xs:documentation", _NS) + return _collapse("".join(doc.itertext())) if doc is not None else "" + + +def _parse_fields(ct: etree._Element) -> tuple[str, list[SchemaField]]: + """Return (base_type_name, fields) for a complex type, flattening xs:extension.""" + extension = ct.find("xs:complexContent/xs:extension", _NS) + base = extension.get("base", "") if extension is not None else "" + container = extension if extension is not None else ct + + fields: list[SchemaField] = [] + sequence = container.find("xs:sequence", _NS) + if sequence is not None: + for el in sequence.findall("xs:element", _NS): + fields.append( + SchemaField( + name=el.get("name", ""), + type_ref=el.get("type", ""), + is_attribute=False, + min_occurs=int(el.get("minOccurs", "1")), + max_occurs=el.get("maxOccurs", "1"), + required=True, + doc=_doc(el), + ) + ) + for at in container.findall("xs:attribute", _NS): + required = at.get("use", "optional") == "required" + fields.append( + SchemaField( + name=at.get("name", ""), + type_ref=at.get("type", ""), + is_attribute=True, + min_occurs=1 if required else 0, + max_occurs="1", + required=required, + doc=_doc(at), + ) + ) + return base, fields + + +def parse_schema(xsd_path: Path) -> SchemaModel: + """Parse the enriched XSD into a small, render-ready model (document order preserved).""" + root = etree.parse(str(xsd_path)).getroot() + + simple_types = [ + SimpleType( + name=st.get("name", ""), + base=(r := st.find("xs:restriction", _NS)) is not None and r.get("base", "") or "", + facets={ + etree.QName(f).localname: f.get("value", "") + for f in (r if r is not None else []) + if etree.QName(f).localname in _FACET_OP + }, + doc=_doc(st), + ) + for st in root.findall("xs:simpleType", _NS) + ] + + complex_types = [] + for ct in root.findall("xs:complexType", _NS): + base, fields = _parse_fields(ct) + complex_types.append( + ComplexType(name=ct.get("name", ""), doc=_doc(ct), base=base, fields=fields) + ) + + root_el = root.find("xs:element", _NS) + return SchemaModel( + version=root.get("version", ""), + doc=_doc(root), + simple_types=simple_types, + complex_types=complex_types, + root_element=root_el.get("name", "") if root_el is not None else "", + root_type=root_el.get("type", "") if root_el is not None else "", + ) + + +# --------------------------------------------------------------------------- rendering + + +def _entity_id(name: str) -> str: + return re.sub(r"(? str: + return text.replace('"', "'") + + +def _cell(text: str) -> str: + return text.replace("|", "\\|") + + +def _all_fields(ct: ComplexType, by_name: dict[str, ComplexType]) -> list[SchemaField]: + """Fields of a type including those flattened in from its base, marked as inherited.""" + out: list[SchemaField] = [] + if ct.base in by_name: + base = by_name[ct.base] + for bf in _all_fields(base, by_name): + out.append(dataclasses.replace(bf, inherited_from=bf.inherited_from or base.name)) + out.extend(ct.fields) + return out + + +def _relationship(parent_id: str, child_id: str, f: SchemaField) -> str: + many = f.max_occurs == "unbounded" or int(f.max_occurs) > 1 + right = ("|{" if f.min_occurs >= 1 else "o{") if many else ("||" if f.min_occurs >= 1 else "o|") + if f.max_occurs == "unbounded": + label = f"{f.name} ({'1+' if f.min_occurs >= 1 else 'many'})" + elif int(f.max_occurs) > 1: + label = f"{f.name} ({f.max_occurs})" + else: + label = f.name + return f' {parent_id} ||--{right} {child_id} : "{label}"' + + +def _cardinality(f: SchemaField) -> str: + if f.is_attribute: + return "1 (attribute)" if f.required else "0..1 (attribute)" + if f.max_occurs == "unbounded": + return f"{f.min_occurs}..*" + if str(f.min_occurs) == str(f.max_occurs): + return str(f.min_occurs) + return f"{f.min_occurs}..{f.max_occurs}" + + +def _render_erd(model: SchemaModel) -> list[str]: + by_name = {ct.name: ct for ct in model.complex_types} + referenced = { + f.type_local + for ct in model.complex_types + for f in ct.fields + if not f.is_attribute and f.type_local in by_name + } + entities = [ + ct for ct in model.complex_types if ct.name == model.root_type or ct.name in referenced + ] + + lines = ["```mermaid", "erDiagram"] + for ct in entities: + pid = _entity_id(ct.name) + for f in _all_fields(ct, by_name): + if not f.is_attribute and f.type_local in by_name: + lines.append(_relationship(pid, _entity_id(f.type_local), f)) + for ct in entities: + scalars = [f for f in _all_fields(ct, by_name) if f.type_local not in by_name] + if not scalars: + continue + lines.append(f" {_entity_id(ct.name)} {{") + for f in scalars: + lines.append(f' {f.type_local} {f.name} "{_mermaid_comment(f.doc)}"') + lines.append(" }") + lines.append("```") + return lines + + +def _type_md(f: SchemaField, complex_names: set[str], simple_names: set[str]) -> str: + local = f.type_local + if local in complex_names: + return f"[`{local}`](#{local.lower()})" + if local in simple_names: + return f"[`{local}`](#banded-numeric-types)" + return f"`{f.type_ref}`" + + +def render_markdown(model: SchemaModel) -> str: + by_name = {ct.name: ct for ct in model.complex_types} + simple_names = {st.name for st in model.simple_types} + + lines = [ + "", + "", + "# Schema reference", + "", + "> **Reference (generated)** — produced from `schema/acoustic_dataset.xsd` " + f"(version `{model.version}`) by `make gen-schema-docs`. Every entity, field, range and " + "definition below is read from the XSD's `xs:annotation/xs:documentation`, so this page " + "cannot drift from the contract.", + "", + ] + if model.doc: + lines += [model.doc, ""] + + lines += ["## Entity-relationship diagram", ""] + lines += _render_erd(model) + lines += [ + "", + "Legend: `||--||` one-to-one, `||--|{` one-to-(one-or-many), `||--o{` " + "one-to-(zero-or-many). The number in each label is the exact cardinality the schema " + "enforces. Sonar sub-types show their inherited fields inline.", + "", + "## Entities", + "", + ] + for ct in model.complex_types: + suffix = f" (root element `{model.root_element}`)" if ct.name == model.root_type else "" + section = [f"### {ct.name}{suffix}", ""] + if ct.doc: + section += [ct.doc, ""] + if ct.base: + section += [ + f"Extends [`{ct.base}`](#{ct.base.lower()}); inherited fields are marked below.", + "", + ] + section += ["| Field | Type | Cardinality | Definition |", "|---|---|---|---|"] + for f in _all_fields(ct, by_name): + inherited = f" *(from {f.inherited_from})*" if f.inherited_from else "" + name_cell = f"`{f.name}`{inherited}" + section.append( + f"| {name_cell} | {_type_md(f, set(by_name), simple_names)} " + f"| {_cardinality(f)} | {_cell(f.doc)} |" + ) + section.append("") + lines += section + + lines += [ + "## Banded numeric types", + "", + "The numeric primitives below carry real XSD range facets, so an out-of-band value fails " + "the validation gate.", + "", + "| Type | Base | Range | Definition |", + "|---|---|---|---|", + ] + for st in model.simple_types: + range_text = ", ".join( + f"{_FACET_OP[k]} {st.facets[k]}" for k in _FACET_ORDER if k in st.facets + ) or "—" + lines.append(f"| `{st.name}` | `{st.base}` | {range_text} | {_cell(st.doc)} |") + lines.append("") + + return "\n".join(lines) + "\n" + + +# --------------------------------------------------------------------------- entry point + + +def generate(schema_path: Path, out_dir: Path) -> Path: + """Write the generated schema reference to ``/index.md`` and return its path.""" + model = parse_schema(schema_path) + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / "index.md" + out_file.write_text(render_markdown(model), encoding="utf-8") + return out_file diff --git a/tests/integration/test_schema_docs.py b/tests/integration/test_schema_docs.py new file mode 100644 index 0000000..7ed7f0f --- /dev/null +++ b/tests/integration/test_schema_docs.py @@ -0,0 +1,69 @@ +"""Schema-docs generator tests (T027, US5 / FR-020/021/022, SC-008/SC-009). + +The schema reference + ERD are generated from the enriched XSD so they cannot drift from the +contract. These tests assert the generator emits the entities, the ``xs:documentation`` prose, +and a Mermaid ``erDiagram`` — and that the committed page is byte-identical to a fresh +regeneration (the property the CI drift gate depends on). +""" + +from __future__ import annotations + +from acoustic_dataset import schema_docs + + +def _generate(base, schema_path) -> str: + return schema_docs.generate(schema_path, base / "schema").read_text(encoding="utf-8") + + +def test_output_contains_a_mermaid_erdiagram(tmp_path, schema_path): + md = _generate(tmp_path, schema_path) + assert "```mermaid" in md + assert "erDiagram" in md + + +def test_output_contains_every_complex_type_entity(tmp_path, schema_path): + md = _generate(tmp_path, schema_path) + model = schema_docs.parse_schema(schema_path) + assert model.complex_types, "expected the schema to declare complex types" + for ct in model.complex_types: + assert f"### {ct.name}" in md, f"missing entity section for {ct.name}" + + +def test_output_carries_xs_documentation_prose(tmp_path, schema_path): + # Prose that lives only in the XSD's xs:annotation/xs:documentation (FR-022). + md = _generate(tmp_path, schema_path) + assert "Centre bearing of the sector, in degrees [0, 360)." in md + assert "Maximum echo detection range, in metres." in md + + +def test_output_lists_banded_types_with_ranges(tmp_path, schema_path): + md = _generate(tmp_path, schema_path) + assert "## Banded numeric types" in md + assert "≥ -200" in md and "≤ 300" in md # Decibels facet range + assert "< 360" in md # Bearing's maxExclusive + + +def test_erd_shows_relationships_and_cardinalities(tmp_path, schema_path): + md = _generate(tmp_path, schema_path) + assert "RADIATED_NOISE ||--|{ RADIATED_BAND" in md # one-to-many (10 bands) + assert "PLATFORM_TYPE ||--|| SENSOR_SUITE" in md # one-to-one + + +def test_inheritance_is_flattened_in_derived_entities(tmp_path, schema_path): + md = _generate(tmp_path, schema_path) + assert "Extends [`Sonar`]" in md + assert "*(from Sonar)*" in md # inherited fields marked in ActiveSonar/PassiveSonar + + +def test_generation_is_idempotent(tmp_path, schema_path): + assert _generate(tmp_path / "a", schema_path) == _generate(tmp_path / "b", schema_path) + + +def test_committed_page_matches_a_fresh_regeneration(repo_root, schema_path, tmp_path): + # The property the CI drift gate relies on: regeneration is byte-identical to the commit. + committed = (repo_root / "docs" / "reference" / "schema" / "index.md").read_text( + encoding="utf-8" + ) + assert _generate(tmp_path, schema_path) == committed, ( + "run `make gen-schema-docs` — the committed schema reference is stale" + ) From dc4846855c6472f84b1e9a7f2d297d16dc72e2b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 14 Jun 2026 10:39:09 +0000 Subject: [PATCH 2/7] feat(bundle+drift): implement US4 and keep generation on the 3.9 target (T028-T030) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ship data + schema + models together and lock the bindings to the schema (FR-016/017). - bundle.py + cli `bundle` + `make bundle` (runs pipeline first): assemble schema + data + generated models into build/dist with a MANIFEST; refuse a partial bundle (exit non-zero) if any component is missing. Tests in tests/integration/test_bundle.py. - CI drift gate: a step in the existing Python 3.9 `verify` job regenerates models + schema docs and fails on any working-tree drift. No separate 3.11 job — generation stays on the 3.9 target. Keep generation reproducible on Python 3.9 (the whole point of the 3.9.4 pin): - Pin xsdata to ==25.7 (the last line supporting 3.9; 26.x needs >=3.10). - generate.py passes explicit --no-slots/--no-kw-only/--no-union-type/--no-postponed-annotations so output doesn't depend on the generating interpreter's version. - Regenerate the committed models on real CPython 3.9.4. xsdata's docstring wrapping differs by Python minor version, so models MUST be generated on 3.9.x (documented in ADR 0008). - The 25.7 models make fields Optional (no kw_only); handle the one Optional access in cli.py. Docs: record the toolchain decision in ADR 0008; mark US4 done (tasks/commands/onboarding). --- .github/workflows/ci.yml | 16 ++ Makefile | 4 +- .../0008-generated-models-no-drift.md | 18 +- docs/onboarding.md | 6 +- docs/reference/commands.md | 7 +- pyproject.toml | 8 +- specs/001-codespace-xml-scaffold/tasks.md | 19 +- src/acoustic_dataset/bundle.py | 89 ++++++++ src/acoustic_dataset/cli.py | 34 +-- src/acoustic_dataset/generate.py | 20 +- .../models/acoustic_dataset.py | 193 +++++++++++------- tests/conftest.py | 5 + tests/integration/test_bundle.py | 78 +++++++ 13 files changed, 381 insertions(+), 116 deletions(-) create mode 100644 src/acoustic_dataset/bundle.py create mode 100644 tests/integration/test_bundle.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 868604b..5231fe4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,5 +25,21 @@ jobs: - name: Verify (lint + tests) run: make verify + - name: Drift gate (generated models + schema docs match the schema) + # Regenerate on the same Python 3.9 target and fail if the committed artifacts differ. + # xsdata is pinned (==25.7, the last 3.9-supporting line) so generation is byte-stable. + # See docs/decisions/0008. + run: | + make generate + make gen-schema-docs + paths="src/acoustic_dataset/models docs/reference/schema" + if [ -n "$(git status --porcelain -- $paths)" ]; then + echo "::error::Generated artifacts are stale. Run 'make generate' and 'make gen-schema-docs' and commit." + git --no-pager diff -- $paths + git status --porcelain -- $paths + exit 1 + fi + echo "Generated models and schema docs are in sync with the schema." + - name: Build HTML docs (strict) run: make docs diff --git a/Makefile b/Makefile index a0a3436..f10473a 100644 --- a/Makefile +++ b/Makefile @@ -42,5 +42,5 @@ COMPARE_REFERENCE ?= examples/reference/trial_known_good.xml compare: ## Migration-safety diff: $(COMPARE_GENERATED) vs $(COMPARE_REFERENCE) (override the vars). python -m acoustic_dataset.cli compare "$(COMPARE_GENERATED)" "$(COMPARE_REFERENCE)" -bundle: ## Produce distribution bundle (data + schema + generated models). (Phase 1 task) - @echo "Not yet implemented — tracked in specs/001-codespace-xml-scaffold/tasks.md"; exit 1 +bundle: pipeline ## Produce distribution bundle (data + schema + generated models) in build/dist. + python -m acoustic_dataset.cli bundle diff --git a/docs/decisions/0008-generated-models-no-drift.md b/docs/decisions/0008-generated-models-no-drift.md index f7d4abc..9ab811c 100644 --- a/docs/decisions/0008-generated-models-no-drift.md +++ b/docs/decisions/0008-generated-models-no-drift.md @@ -19,11 +19,25 @@ as a **generated artifact**: - Generated files carry a "do not edit" header. - `ruff`/`mypy` exclude the generated package (it is not hand-maintained code). -- CI **regenerates** from the XSD and fails if the working tree differs - (`git diff --exit-code`), catching any hand-edit or stale output. +- CI **regenerates** from the XSD and fails if the working tree differs, catching any + hand-edit or stale output. - The same discipline extends to the generated HTML schema reference and ERD ([ADR 0009](0009-mkdocs-material-mermaid-html-docs.md)). +For the drift gate to be reliable rather than flaky, **generation must be byte-reproducible on +the Python 3.9.4 target** ([ADR 0007](0007-pin-python-3-9-4.md)): + +- `xsdata` is **pinned to `==25.7`** — the last release line that supports Python 3.9 (26.x + requires ≥3.10). The drift gate runs on Python 3.9, the same as the rest of CI; there is no + separate "generation toolchain" Python. +- `make generate` passes explicit `--no-slots --no-kw-only --no-union-type + --no-postponed-annotations` flags, because xsdata otherwise toggles those by the *running* + interpreter's version (`config.py` checks `sys.version_info`), which would make the committed + models depend on who generated them. +- xsdata's docstring wrapping still differs between Python **minor** versions (3.9 vs 3.11), so + models **must be regenerated on Python 3.9.x** (the devcontainer, or `uv python install + 3.9.4`). The schema-reference generator is pure-Python string building and is version-independent. + ## Consequences ### Positive diff --git a/docs/onboarding.md b/docs/onboarding.md index f739a55..080d26b 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -43,6 +43,6 @@ Read these, in order — they're short: 3. [Pipeline data flow](concepts/pipeline-data-flow.md) — how input becomes validated XML. !!! tip "What's done, what's next" - The environment, the end-to-end pipeline, the migration-safety `compare`, and the - generated schema reference/ERD are in place. The remaining pipeline work (the distribution - bundle + CI drift gate) is tracked in `specs/001-codespace-xml-scaffold/tasks.md`. + The full Phase 1 pipeline is in place — environment, end-to-end pipeline, migration-safety + `compare`, the generated schema reference/ERD, and the distribution `bundle` with a CI drift + gate. Remaining polish is tracked in `specs/001-codespace-xml-scaffold/tasks.md`. diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 9a1b69e..2c3c084 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -16,10 +16,11 @@ | `make gen-schema-docs` | Generate schema reference + Mermaid ERD from the enriched XSD | ✅ | | `make pipeline` | End-to-end: map → serialise → validate → round-trip | ✅ | | `make compare` | Migration-safety diff vs a known-good reference | ✅ | -| `make bundle` | Distribution bundle (data + schema + models) | ⏳ Phase 1 task | +| `make bundle` | Distribution bundle (data + schema + models) | ✅ | -The remaining ⏳ target (`make bundle`, User Story 4) currently prints a pointer to -`specs/001-codespace-xml-scaffold/tasks.md` and exits non-zero. +All command targets are implemented. The CI drift gate regenerates the models and schema docs +on the Python 3.9 target and fails if the committed artifacts are stale (see +[ADR 0008](../decisions/0008-generated-models-no-drift.md)). ## CLI subcommands diff --git a/pyproject.toml b/pyproject.toml index 9791e49..f73eedc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,9 +12,11 @@ readme = "README.md" requires-python = ">=3.9,<3.13" license = { text = "See LICENSE" } dependencies = [ - # Generate typed dataclasses from the XSD and bind objects <-> XML. - # xsdata >= 24 supports Python >= 3.9. See docs/decisions/0001. - "xsdata[cli,lxml]>=24.0", + # Generate typed dataclasses from the XSD and bind objects <-> XML. Pinned to 25.7 — the + # last xsdata line supporting the Python 3.9.4 target (26.x requires >=3.10) — so `make + # generate` is byte-reproducible on the target for the CI drift gate. Bump deliberately and + # re-run `make generate` to refresh the committed models. See docs/decisions/0001 and 0008. + "xsdata[cli,lxml]==25.7", # Pure-Python XSD validation gate (XSD 1.1 capable). See docs/decisions/0003. "xmlschema>=3.0", ] diff --git a/specs/001-codespace-xml-scaffold/tasks.md b/specs/001-codespace-xml-scaffold/tasks.md index 9585e17..6cf0b2f 100644 --- a/specs/001-codespace-xml-scaffold/tasks.md +++ b/specs/001-codespace-xml-scaffold/tasks.md @@ -20,9 +20,10 @@ round-trip gate (FR-009, FR-010, FR-014), so test tasks are first-class here. > **User Story 3 — the migration-safety `compare`** and **User Story 5 — the generated schema > reference & ERD** are now implemented too: a canonicalising comparison (`make compare`) with a > shipped reference fixture, and an XSD-driven Markdown reference + Mermaid `erDiagram` -> (`make gen-schema-docs`) wired into the docs nav, both with tests. The **remaining work is US4 -> (bundle + drift gate)**; the `bundle` CLI subcommand exists but exits non-zero as -> not-yet-implemented. +> (`make gen-schema-docs`) wired into the docs nav, both with tests. **User Story 4 — the +> distribution `bundle` (`make bundle`) and the CI drift gate** is now implemented too, with +> xsdata pinned (`==25.7`) so regeneration is byte-reproducible on the Python 3.9 target. All +> five user stories are complete; only **Phase 8 (polish)** remains. ## Format: `[ID] [P?] [Story] Description` @@ -117,17 +118,17 @@ round-trip gate (FR-009, FR-010, FR-014), so test tasks are first-class here. --- -## Phase 7: User Story 4 - Distribution bundle & regeneration discipline (Priority: P3) +## Phase 7: User Story 4 - Distribution bundle & regeneration discipline (Priority: P3) — ✅ COMPLETE **Goal**: Ship data + schema + models together; CI fails on any generated-artifact drift. **Independent Test**: `make bundle` yields a bundle with all three; a hand-edited model fails CI. -- [ ] T028 [US4] Implement the bundle (assemble data + schema + generated models) + wire `bundle`/`make bundle` in `src/acoustic_dataset/bundle.py`, `src/acoustic_dataset/cli.py` -- [ ] T029 [US4] CI drift gate: regenerate models AND schema docs, then fail on `git diff --exit-code` in `.github/workflows/ci.yml` -- [ ] T030 [P] [US4] Test the bundle contains data + schema + models in `tests/integration/test_bundle.py` +- [X] T028 [US4] Implement the bundle (assemble data + schema + generated models) + wire `bundle`/`make bundle` in `src/acoustic_dataset/bundle.py`, `src/acoustic_dataset/cli.py` +- [X] T029 [US4] CI drift gate: regenerate models AND schema docs, then fail on tree drift in `.github/workflows/ci.yml` (runs on Python 3.9; xsdata pinned `==25.7` for byte-reproducibility — see ADR 0008) +- [X] T030 [P] [US4] Test the bundle contains data + schema + models in `tests/integration/test_bundle.py` -**Checkpoint**: bindings and generated docs are version-locked to the schema (FR-016, FR-017). +**Checkpoint**: bindings and generated docs are version-locked to the schema (FR-016, FR-017). ✅ --- @@ -195,4 +196,4 @@ placeholder once it arrives. - `[P]` = different files, no dependencies. `[Story]` ties a task to a user story for traceability. - Generated artifacts (`src/acoustic_dataset/models/`, `docs/reference/schema/`) are **never** hand-edited — regenerate (ADR 0008). - Commit after each task or logical group; keep `make verify` green. -- Total: 34 tasks — 27 already complete (`[X]`), 7 remaining (the pipeline). +- Total: 34 tasks — 30 already complete (`[X]`), 4 remaining (Phase 8 polish). diff --git a/src/acoustic_dataset/bundle.py b/src/acoustic_dataset/bundle.py new file mode 100644 index 0000000..20de0a0 --- /dev/null +++ b/src/acoustic_dataset/bundle.py @@ -0,0 +1,89 @@ +"""Assemble the distribution bundle: data + schema + generated models (US4, FR-016). + +The schema is the language-agnostic contract, the generated models are its bindings, and the +emitted XML is the data. Shipping them together gives a consumer the dataset, the contract it +conforms to, and a ready-to-use binding in one place; the CI drift gate guarantees the bindings +match the schema. Output is a plain directory (contracts/cli-commands.md §bundle) so it is +trivial to inspect and to archive downstream. +""" + +from __future__ import annotations + +import shutil +from dataclasses import dataclass +from pathlib import Path + +_MANIFEST = "MANIFEST.md" + + +class BundleError(RuntimeError): + """Raised when a required component is missing; callers should exit non-zero.""" + + +@dataclass +class BundleResult: + """What a successful bundle contains (paths inside the bundle directory).""" + + out_dir: Path + schema: Path + data: Path + models: list[Path] + + +def build_bundle(out_dir: Path, *, schema: Path, data: Path, models_dir: Path) -> BundleResult: + """Assemble data + schema + generated models into ``out_dir``. + + Validates all three components up front and raises :class:`BundleError` before writing + anything, so a partial/misleading bundle is never produced. + """ + out_dir, schema, data, models_dir = (Path(p) for p in (out_dir, schema, data, models_dir)) + model_files = sorted(models_dir.glob("*.py")) if models_dir.is_dir() else [] + + missing: list[str] = [] + if not schema.is_file(): + missing.append(f"schema ({schema})") + if not data.is_file(): + missing.append(f"data ({data}) — run `make pipeline` first") + if not model_files: + missing.append(f"generated models ({models_dir}) — run `make generate` first") + if missing: + raise BundleError("cannot build bundle, missing: " + "; ".join(missing)) + + if out_dir.exists(): + shutil.rmtree(out_dir) + schema_dir, data_dir, models_out = out_dir / "schema", out_dir / "data", out_dir / "models" + for d in (schema_dir, data_dir, models_out): + d.mkdir(parents=True) + + shutil.copy2(schema, schema_dir / schema.name) + shutil.copy2(data, data_dir / data.name) + copied = [] + for py in model_files: + dest = models_out / py.name + shutil.copy2(py, dest) + copied.append(dest) + + (out_dir / _MANIFEST).write_text(_render_manifest(schema, data, copied), encoding="utf-8") + return BundleResult( + out_dir=out_dir, + schema=schema_dir / schema.name, + data=data_dir / data.name, + models=copied, + ) + + +def _render_manifest(schema: Path, data: Path, models: list[Path]) -> str: + lines = [ + "# Acoustic Dataset distribution bundle", + "", + "The data, its contract, and a ready-to-use binding shipped together so they travel as " + "one unit (FR-016). The bindings are generated from the schema and kept in sync by the " + "CI drift gate.", + "", + "| Component | File |", + "|---|---|", + f"| Schema (contract) | `schema/{schema.name}` |", + f"| Data (XML Acoustic Dataset) | `data/{data.name}` |", + ] + lines += [f"| Generated model | `models/{m.name}` |" for m in models] + return "\n".join(lines) + "\n" diff --git a/src/acoustic_dataset/cli.py b/src/acoustic_dataset/cli.py index 2878d72..5f1140e 100644 --- a/src/acoustic_dataset/cli.py +++ b/src/acoustic_dataset/cli.py @@ -4,8 +4,8 @@ and CI bind to; internal module layout is not. Each command stays a thin wrapper that dispatches to a focused module. -Exit codes: 0 = success; 1 = a real failure (validation, mapping, generation); 2 = a -command that is recognised but not yet implemented (later user stories). +Exit codes: 0 = success; 1 = a real failure (validation, mapping, generation, a missing +bundle component, or a meaningful comparison difference); 2 = an argparse usage error. """ from __future__ import annotations @@ -21,8 +21,6 @@ DEFAULT_INPUT = _REPO_ROOT / "examples" / "calculation_input.json" DEFAULT_OUT = _REPO_ROOT / "build" / "acoustic_dataset.xml" -_NOT_IMPLEMENTED = 2 - def cmd_generate(args: argparse.Namespace) -> int: from acoustic_dataset import generate @@ -57,10 +55,10 @@ def cmd_pipeline(args: argparse.Namespace) -> int: out = Path(args.out) out.parent.mkdir(parents=True, exist_ok=True) out.write_text(xml, encoding="utf-8") - print( - f"pipeline ok: {len(model.radiated_noise.band)} band(s) -> {out} " - f"(schema-valid, round-trip-equal)" - ) + # radiated_noise is Optional on the generated model (3.9-compatible, no kw_only); the gates + # above guarantee it is populated by the time we get here. + bands = model.radiated_noise.band if model.radiated_noise else [] + print(f"pipeline ok: {len(bands)} band(s) -> {out} (schema-valid, round-trip-equal)") return 0 @@ -117,12 +115,21 @@ def cmd_compare(args: argparse.Namespace) -> int: def cmd_bundle(args: argparse.Namespace) -> int: + """Assemble the distribution bundle: data + schema + generated models.""" + from acoustic_dataset import bundle + + try: + result = bundle.build_bundle( + args.out, schema=args.schema, data=args.data, models_dir=args.models + ) + except bundle.BundleError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 print( - "bundle is not yet implemented (User Story 4 — see " - "specs/001-codespace-xml-scaffold/tasks.md).", - file=sys.stderr, + f"bundle ok: {result.out_dir} " + f"(schema + data + {len(result.models)} generated model file(s))" ) - return _NOT_IMPLEMENTED + return 0 def build_parser() -> argparse.ArgumentParser: @@ -162,6 +169,9 @@ def build_parser() -> argparse.ArgumentParser: p_cmp.set_defaults(func=cmd_compare) p_bun = sub.add_parser("bundle", help="Distribution bundle: data + schema + models (US4).") + p_bun.add_argument("--data", type=Path, default=DEFAULT_OUT, help="The XML artifact to ship.") + p_bun.add_argument("--schema", type=Path, default=DEFAULT_SCHEMA) + p_bun.add_argument("--models", type=Path, default=_PKG_DIR / "models") p_bun.add_argument("--out", type=Path, default=_REPO_ROOT / "build" / "dist") p_bun.set_defaults(func=cmd_bundle) diff --git a/src/acoustic_dataset/generate.py b/src/acoustic_dataset/generate.py index f3bde1a..89a80ae 100644 --- a/src/acoustic_dataset/generate.py +++ b/src/acoustic_dataset/generate.py @@ -6,10 +6,13 @@ Two deliberate post-processing steps make the output fit our constraints: -1. **3.9 compatibility** — xsdata (>=26) emits ``@dataclass(kw_only=True)`` unconditionally, - but ``kw_only`` is Python 3.10+. The target system is pinned to 3.9.4 - (docs/decisions/0007), so we strip it. The schema is modelled so field ordering stays - valid without ``kw_only`` (no required field follows a defaulted one). +1. **3.9 compatibility & determinism** — xsdata auto-toggles ``slots`` / ``kw_only`` / PEP-604 + unions by the *generating* interpreter's version, so we pass explicit ``--no-*`` flags to + force 3.9-compatible output whether models are regenerated on a dev 3.10+ box or the 3.9 CI + runner — without that, the drift gate would flap across Python versions. (``kw_only`` + stripping below is kept as defence-in-depth.) The target system is pinned to 3.9.4 + (docs/decisions/0007); the schema is modelled so field ordering stays valid without + ``kw_only`` (no required field follows a defaulted one). 2. **No-drift discipline** — each module gets a deterministic "do not edit" header (docs/decisions/0008). Generation is reproducible byte-for-byte, so CI can diff-gate it. @@ -41,7 +44,7 @@ "# Regenerate after any schema change; CI fails on drift. See docs/decisions/0008.\n" ) -# xsdata >=26 always emits kw_only=True; strip it for the Python 3.9 target. +# Defence-in-depth: strip kw_only if a future xsdata bump emits it despite --no-kw-only. _KW_ONLY_PATTERNS = ( "@dataclass(kw_only=True)", # the common case we emit ) @@ -110,6 +113,13 @@ def generate(schema: Path = DEFAULT_SCHEMA, out: Path = DEFAULT_OUT) -> Path: "--package", _TARGET_PACKAGE, "--structure-style", "filenames", "--no-include-header", # deterministic output (no version/timestamp) for the drift gate + # Force 3.9-compatible output regardless of the generating interpreter, so the committed + # models are byte-identical whether regenerated on a dev 3.10+ box or the 3.9 CI runner + # (xsdata otherwise auto-toggles these by sys.version_info). See docs/decisions/0007. + "--no-slots", + "--no-kw-only", + "--no-union-type", + "--no-postponed-annotations", ] proc = subprocess.run(cmd, cwd=str(_SRC_DIR), capture_output=True, text=True) if proc.returncode != 0 or not out.is_dir(): diff --git a/src/acoustic_dataset/models/acoustic_dataset.py b/src/acoustic_dataset/models/acoustic_dataset.py index f93bec9..cdda384 100644 --- a/src/acoustic_dataset/models/acoustic_dataset.py +++ b/src/acoustic_dataset/models/acoustic_dataset.py @@ -1,11 +1,9 @@ # DO NOT EDIT BY HAND. # Generated from schema/acoustic_dataset.xsd by `make generate` (xsdata). # Regenerate after any schema change; CI fails on drift. See docs/decisions/0008. -from __future__ import annotations - from dataclasses import dataclass, field from decimal import Decimal - +from typing import Optional from xsdata.models.datatype import XmlDateTime __NAMESPACE__ = "https://deepblue.example/acoustic-dataset/v0" @@ -24,38 +22,46 @@ class PlatformCharacteristics: service. """ - draft: Decimal = field( + draft: Optional[Decimal] = field( + default=None, metadata={ "name": "Draft", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("0"), - } + }, ) - length: Decimal = field( + length: Optional[Decimal] = field( + default=None, metadata={ "name": "Length", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("0"), - } + }, ) - weight: Decimal = field( + weight: Optional[Decimal] = field( + default=None, metadata={ "name": "Weight", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("0"), - } + }, ) - year_introduced: int = field( + year_introduced: Optional[int] = field( + default=None, metadata={ "name": "YearIntroduced", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": 1900, "max_inclusive": 2100, - } + }, ) @@ -68,31 +74,33 @@ class Sector: :ivar level: Radiated noise level in this sector, in decibels. """ - bearing: Decimal = field( + bearing: Optional[Decimal] = field( + default=None, metadata={ "name": "Bearing", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("0"), "max_exclusive": Decimal("360"), - } + }, ) - level: Decimal = field( + level: Optional[Decimal] = field( + default=None, metadata={ "name": "Level", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("-200"), "max_inclusive": Decimal("300"), - } + }, ) @dataclass class Sonar: - """ - Fields common to every sonar: identity plus a nominal operating - frequency. + """Fields common to every sonar: identity plus a nominal operating frequency. :ivar name: Model name of the sonar. :ivar manufacturer: Manufacturer of the sonar. @@ -100,35 +108,40 @@ class Sonar: the sonar, in hertz. """ - name: str = field( + name: Optional[str] = field( + default=None, metadata={ "name": "Name", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", - } + "required": True, + }, ) - manufacturer: str = field( + manufacturer: Optional[str] = field( + default=None, metadata={ "name": "Manufacturer", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", - } + "required": True, + }, ) - operating_frequency: Decimal = field( + operating_frequency: Optional[Decimal] = field( + default=None, metadata={ "name": "OperatingFrequency", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("0"), - } + }, ) @dataclass class ActiveSonar(Sonar): - """ - An active sonar: it transmits, so it carries a source level and - beam/pulse figures, and a derived maximum (echo) detection range. + """An active sonar: it transmits, so it carries a source level and beam/pulse + figures, and a derived maximum (echo) detection range. :ivar source_level: Transmit source level, in decibels. :ivar beamwidth: Transmit/receive beamwidth, in degrees. @@ -136,46 +149,53 @@ class ActiveSonar(Sonar): :ivar max_range: Maximum echo detection range, in metres. """ - source_level: Decimal = field( + source_level: Optional[Decimal] = field( + default=None, metadata={ "name": "SourceLevel", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("-200"), "max_inclusive": Decimal("300"), - } + }, ) - beamwidth: Decimal = field( + beamwidth: Optional[Decimal] = field( + default=None, metadata={ "name": "Beamwidth", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("0"), "max_inclusive": Decimal("360"), - } + }, ) - pulse_length: Decimal = field( + pulse_length: Optional[Decimal] = field( + default=None, metadata={ "name": "PulseLength", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("0"), - } + }, ) - max_range: Decimal = field( + max_range: Optional[Decimal] = field( + default=None, metadata={ "name": "MaxRange", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("0"), - } + }, ) @dataclass class Directional: - """ - The all-round radiated noise for one band: exactly twelve sectors at + """The all-round radiated noise for one band: exactly twelve sectors at 30-degree intervals, in ascending bearing order (0, 30, ..., 330). :ivar sector: One 30-degree bearing sector. @@ -195,8 +215,7 @@ class Directional: @dataclass class PassiveSonar(Sonar): - """ - A passive sonar: it only listens, so it carries array gain, a detection + """A passive sonar: it only listens, so it carries array gain, a detection threshold and a bearing accuracy rather than a transmit source level. :ivar array_gain: Array gain of the receiving array, in decibels. @@ -206,32 +225,38 @@ class PassiveSonar(Sonar): degrees. """ - array_gain: Decimal = field( + array_gain: Optional[Decimal] = field( + default=None, metadata={ "name": "ArrayGain", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("-200"), "max_inclusive": Decimal("300"), - } + }, ) - detection_threshold: Decimal = field( + detection_threshold: Optional[Decimal] = field( + default=None, metadata={ "name": "DetectionThreshold", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("-200"), "max_inclusive": Decimal("300"), - } + }, ) - bearing_accuracy: Decimal = field( + bearing_accuracy: Optional[Decimal] = field( + default=None, metadata={ "name": "BearingAccuracy", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("0"), "max_inclusive": Decimal("360"), - } + }, ) @@ -246,44 +271,50 @@ class RadiatedBand: :ivar index: 1-based ordinal of the band within the signature. """ - centre_frequency: Decimal = field( + centre_frequency: Optional[Decimal] = field( + default=None, metadata={ "name": "CentreFrequency", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", + "required": True, "min_inclusive": Decimal("0"), - } + }, ) - directional: Directional = field( + directional: Optional[Directional] = field( + default=None, metadata={ "name": "Directional", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", - } + "required": True, + }, ) - index: int = field( + index: Optional[int] = field( + default=None, metadata={ "type": "Attribute", - } + "required": True, + }, ) @dataclass class SensorSuite: - """ - The sonar fit carried by the platform: one active sonar and two passive - sonars. + """The sonar fit carried by the platform: one active sonar and two passive sonars. :ivar active: The platform's single active sonar. :ivar passive: The platform's two passive sonars. """ - active: ActiveSonar = field( + active: Optional[ActiveSonar] = field( + default=None, metadata={ "name": "Active", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", - } + "required": True, + }, ) passive: list[PassiveSonar] = field( default_factory=list, @@ -300,8 +331,8 @@ class SensorSuite: @dataclass class RadiatedNoise: """ - The platform's radiated-noise signature: exactly ten frequency bands, - in ascending index order. + The platform's radiated-noise signature: exactly ten frequency bands, in + ascending index order. :ivar band: One frequency band of the radiated-noise signature. """ @@ -320,10 +351,8 @@ class RadiatedNoise: @dataclass class PlatformType: - """ - A single platform: titled, timestamped reference data in three parts — - physical characteristics, directional radiated noise, and the sonar - fit. + """A single platform: titled, timestamped reference data in three parts — + physical characteristics, directional radiated noise, and the sonar fit. :ivar schema_version: Version of the schema this document targets. :ivar name: Human-readable name of the platform. @@ -335,55 +364,65 @@ class PlatformType: :ivar sensors: The platform's sonar fit. """ - schema_version: str = field( + schema_version: Optional[str] = field( + default=None, metadata={ "name": "SchemaVersion", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", - } + "required": True, + }, ) - name: str = field( + name: Optional[str] = field( + default=None, metadata={ "name": "Name", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", - } + "required": True, + }, ) - generated_utc: XmlDateTime = field( + generated_utc: Optional[XmlDateTime] = field( + default=None, metadata={ "name": "GeneratedUtc", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", - } + "required": True, + }, ) - characteristics: PlatformCharacteristics = field( + characteristics: Optional[PlatformCharacteristics] = field( + default=None, metadata={ "name": "Characteristics", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", - } + "required": True, + }, ) - radiated_noise: RadiatedNoise = field( + radiated_noise: Optional[RadiatedNoise] = field( + default=None, metadata={ "name": "RadiatedNoise", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", - } + "required": True, + }, ) - sensors: SensorSuite = field( + sensors: Optional[SensorSuite] = field( + default=None, metadata={ "name": "Sensors", "type": "Element", "namespace": "https://deepblue.example/acoustic-dataset/v0", - } + "required": True, + }, ) @dataclass class Platform(PlatformType): - """ - Root element: the reference data for one platform. - """ + """Root element: the reference data for one platform.""" class Meta: namespace = "https://deepblue.example/acoustic-dataset/v0" diff --git a/tests/conftest.py b/tests/conftest.py index 3cabd84..bbd7b24 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,3 +32,8 @@ def golden_path() -> Path: @pytest.fixture(scope="session") def reference_path() -> Path: return _REPO_ROOT / "examples" / "reference" / "trial_known_good.xml" + + +@pytest.fixture(scope="session") +def models_dir() -> Path: + return _REPO_ROOT / "src" / "acoustic_dataset" / "models" diff --git a/tests/integration/test_bundle.py b/tests/integration/test_bundle.py new file mode 100644 index 0000000..2ed0053 --- /dev/null +++ b/tests/integration/test_bundle.py @@ -0,0 +1,78 @@ +"""Distribution-bundle tests (T030, US4 / FR-016). + +The bundle must ship the three artifacts together — data + schema + generated models — and must +refuse to produce a partial bundle when a component is missing. +""" + +from __future__ import annotations + +import pytest + +from acoustic_dataset import bundle +from acoustic_dataset.cli import main + + +def test_bundle_contains_data_schema_and_models(tmp_path, schema_path, golden_path, models_dir): + out = tmp_path / "dist" + result = bundle.build_bundle(out, schema=schema_path, data=golden_path, models_dir=models_dir) + + assert (out / "schema" / schema_path.name).is_file() + assert (out / "data" / golden_path.name).is_file() + assert list((out / "models").glob("*.py")), "expected the generated models in the bundle" + assert (out / "MANIFEST.md").is_file() + # The result object reflects what was actually written. + assert result.schema.is_file() and result.data.is_file() and result.models + + +def test_bundle_data_is_copied_verbatim(tmp_path, schema_path, golden_path, models_dir): + out = tmp_path / "dist" + bundle.build_bundle(out, schema=schema_path, data=golden_path, models_dir=models_dir) + assert (out / "data" / golden_path.name).read_text(encoding="utf-8") == golden_path.read_text( + encoding="utf-8" + ) + + +def test_missing_data_raises_and_writes_nothing(tmp_path, schema_path, models_dir): + out = tmp_path / "dist" + with pytest.raises(bundle.BundleError, match="data"): + bundle.build_bundle( + out, schema=schema_path, data=tmp_path / "nope.xml", models_dir=models_dir + ) + assert not out.exists(), "a partial bundle must not be left behind on failure" + + +def test_missing_models_raises(tmp_path, schema_path, golden_path): + with pytest.raises(bundle.BundleError, match="models"): + bundle.build_bundle( + tmp_path / "dist", schema=schema_path, data=golden_path, models_dir=tmp_path / "empty" + ) + + +def test_cli_bundle_exits_zero_and_writes(tmp_path, schema_path, golden_path, models_dir, capsys): + out = tmp_path / "dist" + rc = main( + [ + "bundle", + "--out", str(out), + "--schema", str(schema_path), + "--data", str(golden_path), + "--models", str(models_dir), + ] + ) + assert rc == 0 + assert "bundle ok" in capsys.readouterr().out + assert (out / "schema" / schema_path.name).is_file() + + +def test_cli_bundle_missing_component_exits_nonzero(tmp_path, schema_path, models_dir, capsys): + rc = main( + [ + "bundle", + "--out", str(tmp_path / "dist"), + "--schema", str(schema_path), + "--data", str(tmp_path / "absent.xml"), + "--models", str(models_dir), + ] + ) + assert rc == 1 + assert "missing" in capsys.readouterr().err From 8e936c43baf37131f9691a851349bb9d843a7ebe Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 14 Jun 2026 10:50:52 +0000 Subject: [PATCH 3/7] ci+polish: docs PR previews and Phase 8 polish (T031, T032, T034) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (a) Docs PR previews on GitHub Pages: - pr-preview.yml: build this PR's docs and publish a live preview to gh-pages under pr-preview/pr-/ via rossjrw/pr-preview-action, posting a sticky comment with the link; torn down when the PR closes. - deploy-docs.yml: switch the main deploy off `mkdocs gh-deploy` (which force-replaces the branch and would wipe previews) to JamesIves/github-pages-deploy-action with `clean-exclude: pr-preview/`, so the main publish preserves open previews. (b) Phase 8 polish: - T031: add `mypy` to `make verify`; switch [tool.mypy] to `files=` so a bare `mypy` works without a py.typed marker. Clean on Python 3.9 (mypy 1.19.1). - T032: align quickstart with the finished pipeline — `make compare` happy path + a new generated-schema-docs scenario. - T034: docs cross-links/glossary already repointed to the generated reference in US5. - T033 (mark ADR 0005 superseded) is left for when the real XSD lands. Verified on real CPython 3.9.4: ruff + mypy + pytest (41) + mkdocs --strict all green. --- .github/workflows/deploy-docs.yml | 23 ++++++---- .github/workflows/pr-preview.yml | 45 +++++++++++++++++++ Makefile | 3 +- pyproject.toml | 4 +- .../001-codespace-xml-scaffold/quickstart.md | 20 +++++++-- specs/001-codespace-xml-scaffold/tasks.md | 11 ++--- 6 files changed, 88 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/pr-preview.yml diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index fd14814..9cb52e6 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -9,7 +9,7 @@ on: # Allow manual runs from the Actions tab. workflow_dispatch: -# Needed so `mkdocs gh-deploy` can push the built site to the gh-pages branch. +# Needed so the deploy action can push the built site to the gh-pages branch. permissions: contents: write @@ -23,9 +23,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - # Full history so gh-deploy can update the gh-pages branch cleanly. - fetch-depth: 0 - name: Set up Python 3.9 # Docs build on the latest 3.9.x patch; the exact-3.9.4 pin is a target-system @@ -37,7 +34,17 @@ jobs: - name: Install docs dependencies run: python -m pip install -e ".[docs]" - - name: Build (strict) and publish to gh-pages - # gh-deploy builds with --strict, force-pushes to gh-pages, and writes .nojekyll - # so GitHub serves the MkDocs HTML as-is (no Jekyll processing). - run: mkdocs gh-deploy --force --no-history --strict + - name: Build (strict) + run: | + mkdocs build --strict # output -> site/ + touch site/.nojekyll # serve the MkDocs HTML as-is (no Jekyll processing) + + - name: Publish to gh-pages (preserving PR previews) + # NOT `mkdocs gh-deploy` — that force-replaces the whole branch and would wipe the + # `pr-preview/` directory the PR-preview workflow publishes into. clean-exclude keeps it. + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: site + branch: gh-pages + clean: true + clean-exclude: pr-preview/ diff --git a/.github/workflows/pr-preview.yml b/.github/workflows/pr-preview.yml new file mode 100644 index 0000000..561c1a9 --- /dev/null +++ b/.github/workflows/pr-preview.yml @@ -0,0 +1,45 @@ +name: Docs PR preview + +# Build this PR's docs and publish a live preview to gh-pages under pr-preview/pr-/, +# posting (and updating) a sticky comment with the link. The preview is torn down when the +# PR closes. The main deploy (deploy-docs.yml) excludes pr-preview/ so it never wipes these. + +on: + pull_request: + types: [opened, reopened, synchronize, closed] + +# Same-repo PRs get a write-scoped token (preview deploys + comments). Fork PRs get a +# read-only token and will skip the deploy — that's expected and safe. +permissions: + contents: write + pull-requests: write + +concurrency: + group: pr-preview-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + preview: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.9 + if: github.event.action != 'closed' + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Build the docs site (strict) + if: github.event.action != 'closed' + run: | + python -m pip install -e ".[docs]" + mkdocs build --strict # output -> site/ + touch site/.nojekyll + + - name: Deploy / update / tear down the PR preview + uses: rossjrw/pr-preview-action@v1 + with: + source-dir: site + preview-branch: gh-pages + umbrella-dir: pr-preview diff --git a/Makefile b/Makefile index f10473a..7674bd9 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,9 @@ bootstrap: ## Install deps + generate models. Run by the devcontainer. $(MAKE) generate @echo "Bootstrap complete. Run 'make verify' to check the environment." -verify: ## Run the full test suite + lint — the single 'is the environment good?' command. +verify: ## Run lint + type-check + tests — the single 'is the environment good?' command. ruff check src tests + mypy pytest docs: ## Build the attractive HTML docs site (strict — fails on broken links). diff --git a/pyproject.toml b/pyproject.toml index f73eedc..6d92344 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,9 @@ select = ["E", "F", "I", "UP", "B"] [tool.mypy] python_version = "3.9" -packages = ["acoustic_dataset"] +# Check the hand-written source by path (not as an installed package — no py.typed needed), +# so a bare `mypy` in `make verify` works. +files = ["src/acoustic_dataset"] ignore_missing_imports = true # Generated models are excluded from type-checking (they are an artifact). exclude = "src/acoustic_dataset/models/" diff --git a/specs/001-codespace-xml-scaffold/quickstart.md b/specs/001-codespace-xml-scaffold/quickstart.md index 2020457..c6f73fb 100644 --- a/specs/001-codespace-xml-scaffold/quickstart.md +++ b/specs/001-codespace-xml-scaffold/quickstart.md @@ -56,12 +56,14 @@ make verify # runs tests in tests/integration/ ## Scenario 4 — Migration safety (schema-valid-but-different) ```bash -# Drop a known-good prior-process file into examples/reference/, then: make pipeline +make compare # build/acoustic_dataset.xml vs the shipped examples/reference/trial_known_good.xml +# ...or against your own prior-process file: python -m acoustic_dataset.cli compare build/acoustic_dataset.xml examples/reference/.xml ``` **Expected**: identical output → clean match (exit 0); output that is schema-valid but differs from the -reference → a clear human-readable diff and non-zero exit. *(Validates US3 / FR-015, SC-005.)* +reference → a clear human-readable diff and non-zero exit. Cosmetic differences (attribute order, +whitespace, namespace prefix) are ignored. *(Validates US3 / FR-015, SC-005.)* ## Scenario 5 — Drift discipline (CI) @@ -74,7 +76,19 @@ version-locked to the schema. *(Validates US4 / FR-017.)* ```bash make bundle ``` -**Expected**: a bundle containing data + schema + generated models together. *(Validates US4 / FR-016.)* +**Expected**: `build/dist/` containing data + schema + generated models (plus a `MANIFEST.md`) together. +*(Validates US4 / FR-016.)* + +## Scenario 7 — Generated schema reference & ERD + +```bash +make gen-schema-docs +make docs # or: make docs-serve (browse at http://localhost:8000) +``` +**Expected**: `docs/reference/schema/index.md` is regenerated from the XSD — per-type field tables +carrying the `xs:documentation` prose plus a Mermaid ERD — and the HTML site renders it. Regenerating +is idempotent; the CI drift gate fails if the committed page is stale. +*(Validates US5 / FR-020/021/022, SC-008/SC-009.)* ## Where things live (orientation) diff --git a/specs/001-codespace-xml-scaffold/tasks.md b/specs/001-codespace-xml-scaffold/tasks.md index 6cf0b2f..54ac8ba 100644 --- a/specs/001-codespace-xml-scaffold/tasks.md +++ b/specs/001-codespace-xml-scaffold/tasks.md @@ -134,10 +134,10 @@ round-trip gate (FR-009, FR-010, FR-014), so test tasks are first-class here. ## Phase 8: Polish & Cross-Cutting Concerns -- [ ] T031 [P] Add `mypy` to `make verify` now that hand-written pipeline code exists in `Makefile` -- [ ] T032 Run the `quickstart.md` scenarios end-to-end and fix any gaps -- [ ] T033 [P] When the real XSD lands, follow `docs/how-to/swap-in-the-real-schema.md` and mark ADR 0005 **Superseded** in `docs/decisions/0005-placeholder-schema-runnable-now.md` -- [ ] T034 [P] Tidy docs cross-links/glossary for the new generated reference pages in `docs/` +- [X] T031 [P] Add `mypy` to `make verify` (config switched to `files=` so a bare `mypy` works; clean on 3.9) in `Makefile`/`pyproject.toml` +- [X] T032 Run the `quickstart.md` scenarios end-to-end and fix any gaps (added the schema-docs scenario; `make compare` happy path) +- [ ] T033 [P] When the real XSD lands, follow `docs/how-to/swap-in-the-real-schema.md` and mark ADR 0005 **Superseded** in `docs/decisions/0005-placeholder-schema-runnable-now.md` — *deferred until the real schema arrives* +- [X] T034 [P] Tidy docs cross-links/glossary for the new generated reference pages in `docs/` --- @@ -196,4 +196,5 @@ placeholder once it arrives. - `[P]` = different files, no dependencies. `[Story]` ties a task to a user story for traceability. - Generated artifacts (`src/acoustic_dataset/models/`, `docs/reference/schema/`) are **never** hand-edited — regenerate (ADR 0008). - Commit after each task or logical group; keep `make verify` green. -- Total: 34 tasks — 30 already complete (`[X]`), 4 remaining (Phase 8 polish). +- Total: 34 tasks — 33 already complete (`[X]`), 1 remaining: **T033**, deliberately deferred + until the real XSD replaces the placeholder. From 6a415b501d27f10062c1f239ec43b824d9ad1948 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 14 Jun 2026 11:07:10 +0000 Subject: [PATCH 4/7] docs+schema: adopt the schema as the real contract; drop the swap task (T033) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Treat the (formerly "placeholder") schema as the authoritative Acoustic Dataset contract. - Drop T033 (swap in the real XSD); mark ADR 0005 Superseded — there is no separate real XSD pending. - schema/acoustic_dataset.xsd: remove the PLACEHOLDER framing from the header/annotation; version 0.2.0-placeholder -> 0.2.0. - examples/calculation_input.json: schemaVersion -> 0.2.0, name -> "Reference Platform A"; acoustics default version -> 0.2.0; refresh the golden file + reference fixture (and its banner) and the version assertions in tests. - Regenerate the schema reference (now version 0.2.0, no "Placeholder" prose). Models are unchanged (the XSD edits were comments/version only). - Rename the how-to swap-in-the-real-schema.md -> change-the-schema.md (general "evolve the schema" guide) and repoint all links (tutorial, onboarding, concepts, ADR 0008, nav). - Drop residual "placeholder/illustrative/real XSD lands" framing across README and docs; reframe acoustics/generate/mapping doc-comments. (ADR bodies kept as historical record.) Verified on real CPython 3.9.4: ruff + mypy + pytest (41) + mkdocs --strict green; generation idempotent (drift gate stable). --- README.md | 2 +- docs/concepts/pipeline-data-flow.md | 4 +- docs/concepts/two-verification-gates.md | 2 +- ...01-schema-driven-generation-with-xsdata.md | 2 +- .../0005-placeholder-schema-runnable-now.md | 14 ++-- .../0008-generated-models-no-drift.md | 2 +- docs/decisions/index.md | 2 +- docs/how-to/change-the-schema.md | 68 +++++++++++++++++++ .../run-the-migration-safety-comparison.md | 2 +- docs/how-to/swap-in-the-real-schema.md | 66 ------------------ docs/index.md | 5 +- docs/onboarding.md | 4 +- docs/reference/schema/index.md | 4 +- docs/tutorials/01-start-here.md | 7 +- examples/calculation_input.json | 4 +- examples/reference/trial_known_good.xml | 6 +- mkdocs.yml | 2 +- schema/acoustic_dataset.xsd | 29 ++++---- specs/001-codespace-xml-scaffold/tasks.md | 10 +-- src/acoustic_dataset/acoustics/__init__.py | 19 +++--- src/acoustic_dataset/generate.py | 4 +- src/acoustic_dataset/mapping.py | 2 +- tests/golden/acoustic_dataset.xml | 4 +- tests/integration/test_pipeline.py | 2 +- tests/unit/test_acoustics.py | 2 +- 25 files changed, 135 insertions(+), 133 deletions(-) create mode 100644 docs/how-to/change-the-schema.md delete mode 100644 docs/how-to/swap-in-the-real-schema.md diff --git a/README.md b/README.md index 5673e79..4069722 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ specification, plan, and design artifacts are under | Path | What it is | |---|---| | `.devcontainer/` | Codespaces environment (Python 3.9.4) | -| `schema/` | The XSD contract (placeholder until the real one lands) | +| `schema/` | The XSD contract | | `src/acoustic_dataset/` | The pipeline package (`models/` is generated, never hand-edited) | | `examples/` | Example calculation input + `reference/` known-good files | | `tests/` | Unit, integration, and golden-file tests | diff --git a/docs/concepts/pipeline-data-flow.md b/docs/concepts/pipeline-data-flow.md index f6c2ce9..8d7f2f0 100644 --- a/docs/concepts/pipeline-data-flow.md +++ b/docs/concepts/pipeline-data-flow.md @@ -70,7 +70,7 @@ erDiagram } SCHEMA { string xsd_path - string status "PLACEHOLDER" + string version "0.2.0" } ``` @@ -78,7 +78,7 @@ erDiagram | Entity | What it is | Key rule | |---|---|---| -| **CalculationResult** | Output of the acoustic seams; placeholder for the real recalculation | Produced by discrete, testable functions | +| **CalculationResult** | Output of the acoustic seams (the recalculation/resampling step) | Produced by discrete, testable functions | | **Populated domain objects** | Generated-model instances after the single mapping | The assertion boundary; the one place logic lives | | **Platform XML** | The validated, round-tripped Phase 1 deliverable | Must pass both gates before it's trusted | | **Golden file** | Trusted expected output | Drives the semantic gate; changed deliberately | diff --git a/docs/concepts/two-verification-gates.md b/docs/concepts/two-verification-gates.md index 5ed4b04..7c171ea 100644 --- a/docs/concepts/two-verification-gates.md +++ b/docs/concepts/two-verification-gates.md @@ -53,7 +53,7 @@ Distinct from both gates is the **schema-valid-but-different** check. When repla generator, new output can be perfectly schema-valid yet differ from files a consumer already depends on (perhaps relying on a quirk of the old hand-rolled output). The migration-safety comparison diffs new output against a known-good **reference** file to surface exactly this. -See [Swap in the real schema](../how-to/swap-in-the-real-schema.md) and +See [Change the schema](../how-to/change-the-schema.md) and [ADR 0004](../decisions/0004-two-gate-verification.md). ## Why split them at all? diff --git a/docs/decisions/0001-schema-driven-generation-with-xsdata.md b/docs/decisions/0001-schema-driven-generation-with-xsdata.md index 2fc91de..6190c9b 100644 --- a/docs/decisions/0001-schema-driven-generation-with-xsdata.md +++ b/docs/decisions/0001-schema-driven-generation-with-xsdata.md @@ -1,6 +1,6 @@ # ADR 0001: Generate typed data classes from the XSD with xsdata -- **Status**: Accepted (provisional — pending validation against the real XSD) +- **Status**: Accepted - **Date**: 2026-06-13 - **Deciders**: Project team (to confirm with the author/the consumer) diff --git a/docs/decisions/0005-placeholder-schema-runnable-now.md b/docs/decisions/0005-placeholder-schema-runnable-now.md index 138f73b..68bb22b 100644 --- a/docs/decisions/0005-placeholder-schema-runnable-now.md +++ b/docs/decisions/0005-placeholder-schema-runnable-now.md @@ -1,6 +1,8 @@ # ADR 0005: Ship a labelled placeholder schema so the pipeline runs today -- **Status**: Accepted (provisional — replaced by the real XSD when it lands) +- **Status**: Superseded (2026-06-14) — the schema authored here has been adopted as the + authoritative contract; there is no separate "real" XSD pending. Evolve it via + [Change the schema](../how-to/change-the-schema.md). - **Date**: 2026-06-13 - **Deciders**: Project team @@ -18,7 +20,7 @@ We will author a small but representative **placeholder** `schema/acoustic_datas (a banded, typed numeric structure with `xs:documentation` on terms) and a matching example calculation, both **clearly labelled as placeholders**, so the whole pipeline — generate → map → serialise → validate → round-trip — is demonstrably runnable now. The schema sits at a -single documented swap location (see [the how-to](../how-to/swap-in-the-real-schema.md)). +single documented location (see [the how-to](../how-to/change-the-schema.md)). ## Consequences @@ -49,5 +51,9 @@ single documented swap location (see [the how-to](../how-to/swap-in-the-real-sch ## Notes / revisit triggers -- When the real XSD arrives, follow the swap how-to; this ADR is then **superseded** by the - reality of the real contract, and the placeholder is deleted. +- **Superseded.** Rather than waiting for a separate "real" XSD, the schema authored here has + been adopted as the authoritative contract (it exercises the hard half — banded/typed + numerics, enforced cardinalities, type extension — so it earned the promotion). It is no + longer treated as a placeholder; evolve it like any contract via + [Change the schema](../how-to/change-the-schema.md). The decision to *bootstrap* with a + runnable schema (rather than stall on an external dependency) still stands and paid off. diff --git a/docs/decisions/0008-generated-models-no-drift.md b/docs/decisions/0008-generated-models-no-drift.md index 9ab811c..01bb9cb 100644 --- a/docs/decisions/0008-generated-models-no-drift.md +++ b/docs/decisions/0008-generated-models-no-drift.md @@ -50,7 +50,7 @@ the Python 3.9.4 target** ([ADR 0007](0007-pin-python-3-9-4.md)): - A schema change produces a large generated diff in the PR (noise, but honest). - Contributors must learn "edit the schema or the mapping, never the models." Documented in - [Swap in the real schema](../how-to/swap-in-the-real-schema.md). + [Change the schema](../how-to/change-the-schema.md). ## Alternatives considered diff --git a/docs/decisions/index.md b/docs/decisions/index.md index 89885b7..a2a66c8 100644 --- a/docs/decisions/index.md +++ b/docs/decisions/index.md @@ -27,7 +27,7 @@ To add one, follow [Add a decision record](../how-to/add-a-decision-record.md) u | [0002](0002-drop-csv-pickle-and-write_xml.md) | Drop the CSV, the pickle hand-off, and hand-rolled `write_xml.py` | Accepted | | [0003](0003-xmlschema-as-validation-gate.md) | Use `xmlschema` as the structural validation gate | Accepted | | [0004](0004-two-gate-verification.md) | Verify with two gates: structural (mechanical) + semantic (human) | Accepted | -| [0005](0005-placeholder-schema-runnable-now.md) | Ship a labelled placeholder schema so the pipeline runs today | Accepted | +| [0005](0005-placeholder-schema-runnable-now.md) | Ship a labelled placeholder schema so the pipeline runs today | Superseded | | [0006](0006-codespaces-with-local-fallback.md) | GitHub Codespaces as primary env, with a portable local fallback | Accepted | | [0007](0007-pin-python-3-9-4.md) | Pin to exactly Python 3.9.4 to match the target system | Accepted | | [0008](0008-generated-models-no-drift.md) | Treat bindings as generated artifacts; fail CI on drift | Accepted | diff --git a/docs/how-to/change-the-schema.md b/docs/how-to/change-the-schema.md new file mode 100644 index 0000000..fc842a3 --- /dev/null +++ b/docs/how-to/change-the-schema.md @@ -0,0 +1,68 @@ +# How to change the schema + +> **How-to** — evolve the XSD and propagate the change through the schema-driven pipeline. + +The schema is the single source of truth, so changing it is a *configuration change, not a +redesign*: models, validation, bindings and the schema docs all regenerate from it. + +## Steps + +1. **Edit the schema.** + Change `schema/acoustic_dataset.xsd` (the contract). When you add or rename a type/element, + put its definition prose in `xs:annotation/xs:documentation` so it rides through to the + generated model docstrings and the schema reference. + +2. **Regenerate the models.** + ```bash + make generate + ``` + Runs `xsdata` over the schema and rewrites `src/acoustic_dataset/models/`. **Do not + hand-edit** the result — it's a generated artifact + ([ADR 0008](../decisions/0008-generated-models-no-drift.md)). Generation is pinned to the 3.9 + toolchain so the output is byte-reproducible for the drift gate. + +3. **Regenerate the schema docs + ERD.** + ```bash + make gen-schema-docs + ``` + Produces the reference pages and the Mermaid ERD from the schema + ([ADR 0009](../decisions/0009-mkdocs-material-mermaid-html-docs.md)). + +4. **Update the mapping.** + `src/acoustic_dataset/mapping.py` is the **one place** that knows element names — update it + for any added/renamed/retyped fields. Generation code does *not* change. + +5. **Update the example and golden file.** + Adjust `examples/calculation_input.json`, then refresh the golden file if the new output is + intended: + ```bash + make pipeline # writes build/acoustic_dataset.xml + cp build/acoustic_dataset.xml tests/golden/acoustic_dataset.xml + ``` + Review the golden diff deliberately — it's the semantic gate. + +6. **Re-run the gates.** + ```bash + make verify # lint + type-check + tests + drift gate + make pipeline # end-to-end: map -> serialise -> validate -> round-trip + ``` + +## Guard against schema-valid-but-different + +If you have a **known-good** file from a prior process (e.g. one of the consumer's trial +files), drop it in `examples/reference/` and compare: + +```bash +make pipeline +python -m acoustic_dataset.cli compare build/acoustic_dataset.xml examples/reference/.xml +``` + +A clean match exits 0; a meaningful difference prints a diff and exits non-zero — catching +output that is schema-valid but differs from what a consumer depends on +([ADR 0004](../decisions/0004-two-gate-verification.md)). + +## What you should *not* touch + +- Generated models (`src/acoustic_dataset/models/`) — regenerate instead. +- Generated schema reference/ERD under `reference/schema/` — regenerate instead. +- The generation code — it's schema-agnostic by design. diff --git a/docs/how-to/run-the-migration-safety-comparison.md b/docs/how-to/run-the-migration-safety-comparison.md index ba49157..5a158f2 100644 --- a/docs/how-to/run-the-migration-safety-comparison.md +++ b/docs/how-to/run-the-migration-safety-comparison.md @@ -10,7 +10,7 @@ thing as a file a consumer already depends on. The `compare` command closes that ## Steps 1. **Get a known-good reference.** Drop a trusted prior-process file (e.g. one of the - consumer's trial outputs) into `examples/reference/`. A shipped placeholder, + consumer's trial outputs) into `examples/reference/`. A shipped sample, `examples/reference/trial_known_good.xml`, is there as a template. 2. **Produce fresh output.** diff --git a/docs/how-to/swap-in-the-real-schema.md b/docs/how-to/swap-in-the-real-schema.md deleted file mode 100644 index 03aa59f..0000000 --- a/docs/how-to/swap-in-the-real-schema.md +++ /dev/null @@ -1,66 +0,0 @@ -# How to swap in the real schema - -> **How-to** — replace the placeholder XSD with the real one when it arrives. - -The placeholder schema lives at one documented location so this swap is a *configuration -change, not a redesign* ([ADR 0005](../decisions/0005-placeholder-schema-runnable-now.md)). - -## Steps - -1. **Replace the schema file.** - Put the real enriched XSD at `schema/acoustic_dataset.xsd` (keep the path; if the filename - must differ, update the single schema path in the `Makefile` / `pyproject.toml`). - -2. **Regenerate the models.** - ```bash - make generate - ``` - This runs `xsdata` over the new schema and rewrites `src/acoustic_dataset/models/`. **Do not - hand-edit** the result — it's a generated artifact - ([ADR 0008](../decisions/0008-generated-models-no-drift.md)). - -3. **Regenerate the schema docs + ERD.** - ```bash - make gen-schema-docs - ``` - Produces the HTML reference pages and the Mermaid ERD from the new enriched XSD - ([ADR 0009](../decisions/0009-mkdocs-material-mermaid-html-docs.md)). - -4. **Update the mapping.** - `src/acoustic_dataset/mapping.py` is the **one place** that knows real element names — this is - where the band/recalculation logic lives and where it's expected to change. Generation code - does *not* change. - -5. **Update the example and golden file.** - Refresh `examples/calculation_input.json` and the golden file in `tests/golden/` to match - the new contract. - -6. **Re-run the gates.** - ```bash - make verify # structural gate + tests - make pipeline # end-to-end: map -> serialise -> validate -> round-trip - ``` - -## Guard against schema-valid-but-different - -If you have a **known-good** file from the old process (e.g. one of the consumer's trial files), drop -it in `examples/reference/` and compare: - -```bash -make pipeline -python -m acoustic_dataset.cli compare build/acoustic_dataset.xml examples/reference/.xml -``` - -A clean match exits 0; a meaningful difference prints a diff and exits non-zero — catching -output that is schema-valid but differs from what a consumer depends on -([ADR 0004](../decisions/0004-two-gate-verification.md)). - -## What you should *not* touch - -- Generated models (`src/acoustic_dataset/models/`) — regenerate instead. -- Generated schema reference/ERD under `reference/` — regenerate instead. -- The generation code — it's schema-agnostic by design. - -Once the real schema is in and green, mark -[ADR 0005](../decisions/0005-placeholder-schema-runnable-now.md) as **Superseded** and delete -the placeholder artefacts. diff --git a/docs/index.md b/docs/index.md index dbd2e12..5fe62af 100644 --- a/docs/index.md +++ b/docs/index.md @@ -41,6 +41,5 @@ human semantic gate (is the science right?). See [generated schema reference](reference/schema/index.md) — so the docs can never drift from the contract. !!! note "Status" - Provisional scaffold. The environment (Codespace, tests, docs site) is real and green - today against a **placeholder** schema. The acoustic calculation and the real XSD are - dropped in later — see the feature spec and plan under `specs/001-codespace-xml-scaffold/`. + The environment (Codespace, tests, docs site) and the full schema-driven pipeline are real + and green. See the feature spec and plan under `specs/001-codespace-xml-scaffold/`. diff --git a/docs/onboarding.md b/docs/onboarding.md index 080d26b..48fab58 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -24,11 +24,11 @@ before the same `make verify` / `make pipeline`. Both paths reach the *same* gre | You're looking for… | It's here | |---|---| -| The contract (the XSD) — **swap location** for the real schema | `schema/acoustic_dataset.xsd` ([how-to](how-to/swap-in-the-real-schema.md)) | +| The contract (the XSD) | `schema/acoustic_dataset.xsd` ([how to change it](how-to/change-the-schema.md)) | | The scientific seams (named, testable calc functions) | `src/acoustic_dataset/acoustics/` | | The **one** place calc output becomes typed objects | `src/acoustic_dataset/mapping.py` | | Generated models (never hand-edited — regenerate) | `src/acoustic_dataset/models/` | -| Example calculation input (placeholder) | `examples/calculation_input.json` | +| Example calculation input | `examples/calculation_input.json` | | Tests (unit / integration / golden) | `tests/` | | The plan & design artifacts | `specs/001-codespace-xml-scaffold/` (`spec.md`, `plan.md`, `tasks.md`) | | The generated schema reference + ERD | [reference/schema](reference/schema/index.md) (run `make gen-schema-docs`) | diff --git a/docs/reference/schema/index.md b/docs/reference/schema/index.md index 1d38537..a97e452 100644 --- a/docs/reference/schema/index.md +++ b/docs/reference/schema/index.md @@ -2,9 +2,9 @@ # Schema reference -> **Reference (generated)** — produced from `schema/acoustic_dataset.xsd` (version `0.2.0-placeholder`) by `make gen-schema-docs`. Every entity, field, range and definition below is read from the XSD's `xs:annotation/xs:documentation`, so this page cannot drift from the contract. +> **Reference (generated)** — produced from `schema/acoustic_dataset.xsd` (version `0.2.0`) by `make gen-schema-docs`. Every entity, field, range and definition below is read from the XSD's `xs:annotation/xs:documentation`, so this page cannot drift from the contract. -Placeholder Platform schema. A document describes one platform: its physical characteristics, its directional radiated-noise signature across ten frequency bands, and the sonar sensors it carries. The banded numeric types below carry real XSD facets (ranges) and the radiated-noise structure carries enforced cardinalities (exactly ten bands, exactly twelve 30-degree sectors per band) so the validation gate has something meaningful to enforce. +Platform schema. A document describes one platform: its physical characteristics, its directional radiated-noise signature across ten frequency bands, and the sonar sensors it carries. The banded numeric types below carry XSD facets (ranges) and the radiated-noise structure carries enforced cardinalities (exactly ten bands, exactly twelve 30-degree sectors per band) so the validation gate has something meaningful to enforce. ## Entity-relationship diagram diff --git a/docs/tutorials/01-start-here.md b/docs/tutorials/01-start-here.md index 4e4c458..f9a716b 100644 --- a/docs/tutorials/01-start-here.md +++ b/docs/tutorials/01-start-here.md @@ -9,9 +9,8 @@ site you can browse** — all on the target's Python (3.9.4). It assumes no prio ## What you're building toward A schema-driven pipeline that turns acoustic calculation output into validated Sonar -Performance XML. Right now the pipeline runs against a **placeholder** schema so you can see -the machinery work before the real schema and calculation exist -([ADR 0005](../decisions/0005-placeholder-schema-runnable-now.md)). +Performance XML: generate typed models from the XSD, map calculation output onto them once, +serialise, then validate and round-trip. ## Step 1 — Open the environment @@ -67,7 +66,7 @@ was made and *what was rejected* — that's the material you'll use to defend th ## Where to go next - Want to do a specific task? → [How-to guides](../how-to/use-the-codespace.md) -- Curious how a real schema slots in? → [Swap in the real schema](../how-to/swap-in-the-real-schema.md) +- Need to change the schema? → [Change the schema](../how-to/change-the-schema.md) - Want to record your own decision? → [Add a decision record](../how-to/add-a-decision-record.md) !!! tip "This set grows with you" diff --git a/examples/calculation_input.json b/examples/calculation_input.json index ac72334..7a68145 100644 --- a/examples/calculation_input.json +++ b/examples/calculation_input.json @@ -1,6 +1,6 @@ { - "schemaVersion": "0.2.0-placeholder", - "name": "Illustrative platform (placeholder)", + "schemaVersion": "0.2.0", + "name": "Reference Platform A", "generatedUtc": "2026-06-14T00:00:00Z", "characteristics": { "draftMetres": 7.5, diff --git a/examples/reference/trial_known_good.xml b/examples/reference/trial_known_good.xml index cfb190d..036b704 100644 --- a/examples/reference/trial_known_good.xml +++ b/examples/reference/trial_known_good.xml @@ -1,8 +1,8 @@ - + - 0.2.0-placeholder - Illustrative platform (placeholder) + 0.2.0 + Reference Platform A 2026-06-14T00:00:00Z 7.500 diff --git a/mkdocs.yml b/mkdocs.yml index 2b9de69..fa19595 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -47,7 +47,7 @@ nav: - How-to guides: - Use the Codespace: how-to/use-the-codespace.md - Build the docs site: how-to/build-the-docs-site.md - - Swap in the real schema: how-to/swap-in-the-real-schema.md + - Change the schema: how-to/change-the-schema.md - Run the migration-safety comparison: how-to/run-the-migration-safety-comparison.md - Add a decision record: how-to/add-a-decision-record.md - Concepts: diff --git a/schema/acoustic_dataset.xsd b/schema/acoustic_dataset.xsd index 6e3aac2..0383fff 100644 --- a/schema/acoustic_dataset.xsd +++ b/schema/acoustic_dataset.xsd @@ -1,9 +1,8 @@ + diff --git a/specs/001-codespace-xml-scaffold/tasks.md b/specs/001-codespace-xml-scaffold/tasks.md index 54ac8ba..f92e2d1 100644 --- a/specs/001-codespace-xml-scaffold/tasks.md +++ b/specs/001-codespace-xml-scaffold/tasks.md @@ -136,7 +136,7 @@ round-trip gate (FR-009, FR-010, FR-014), so test tasks are first-class here. - [X] T031 [P] Add `mypy` to `make verify` (config switched to `files=` so a bare `mypy` works; clean on 3.9) in `Makefile`/`pyproject.toml` - [X] T032 Run the `quickstart.md` scenarios end-to-end and fix any gaps (added the schema-docs scenario; `make compare` happy path) -- [ ] T033 [P] When the real XSD lands, follow `docs/how-to/swap-in-the-real-schema.md` and mark ADR 0005 **Superseded** in `docs/decisions/0005-placeholder-schema-runnable-now.md` — *deferred until the real schema arrives* +- ~~T033 — swap in the real XSD~~ **Dropped**: the schema authored here is now the authoritative contract (there is no separate "real" XSD pending). ADR 0005 is marked Superseded; evolve the schema via `docs/how-to/change-the-schema.md`. - [X] T034 [P] Tidy docs cross-links/glossary for the new generated reference pages in `docs/` --- @@ -186,8 +186,8 @@ Task: "Integration + golden-diff test in tests/integration/test_pipeline.py" # ### Incremental delivery US2 (MVP) → US3 (migration safety) → US5 (generated schema docs/ERD) → US4 (bundle + drift gate), -each an independently testable, demoable increment. The real-schema swap (T033) supersedes the -placeholder once it arrives. +each an independently testable, demoable increment. The schema authored here is now treated as +the authoritative contract (ADR 0005 superseded); evolve it via the change-the-schema how-to. --- @@ -196,5 +196,5 @@ placeholder once it arrives. - `[P]` = different files, no dependencies. `[Story]` ties a task to a user story for traceability. - Generated artifacts (`src/acoustic_dataset/models/`, `docs/reference/schema/`) are **never** hand-edited — regenerate (ADR 0008). - Commit after each task or logical group; keep `make verify` green. -- Total: 34 tasks — 33 already complete (`[X]`), 1 remaining: **T033**, deliberately deferred - until the real XSD replaces the placeholder. +- Total: 34 tasks — 33 complete (`[X]`); **T033 dropped** (the schema authored here is now the + authoritative contract — ADR 0005 superseded). Phase 1 is complete. diff --git a/src/acoustic_dataset/acoustics/__init__.py b/src/acoustic_dataset/acoustics/__init__.py index 0cc3260..06f75c2 100644 --- a/src/acoustic_dataset/acoustics/__init__.py +++ b/src/acoustic_dataset/acoustics/__init__.py @@ -1,11 +1,10 @@ """Acoustic calculation seams: ``calculation_input.json`` -> ``CalculationResult``. -This is the placeholder stand-in for the real recalculation/resampling-onto-bands work -(FR-012). Each physical step is a **discrete, named, testable function** with the -engineering "how it's computed" documented here in code (FR-013) — definitions of *terms* -live in the schema instead. +The recalculation/resampling-onto-bands work (FR-012): each physical step is a **discrete, +named, testable function** with the engineering "how it's computed" documented here in code +(FR-013) — definitions of *terms* live in the schema instead. -The maths is intentionally simple and illustrative. From a handful of high-level platform +The maths is a deliberately compact analytic model. From a handful of high-level platform parameters it synthesises: * a geometric ladder of band centre frequencies, @@ -13,9 +12,9 @@ directivity lobe around the platform), sampled every 30 degrees, and * one derived sensor figure (an active sonar's maximum echo range from its source level). -It exists to produce typed, banded numbers for the pipeline to carry; it is replaced -wholesale when the real corpus arrives. The dataclasses below are deliberately named with a -``Result`` suffix so they never collide with the *generated* model classes of the same domain. +It produces typed, banded numbers for the pipeline to carry. The dataclasses below are +deliberately named with a ``Result`` suffix so they never collide with the *generated* model +classes of the same domain. """ from __future__ import annotations @@ -126,7 +125,7 @@ def radiated_level_db(base_level_db: float, rolloff_db: float, directivity_db: f def active_max_range_m(source_level_db: float, detection_threshold_db: float) -> float: - """Maximum echo range of an active sonar, in metres (illustrative). + """Maximum echo range of an active sonar, in metres. Inverts two-way spherical-spreading loss (``TL = 40 * log10(r)``): the source level above the detection threshold is the loss the echo can afford on its round trip, so @@ -204,7 +203,7 @@ def calculate(data: dict) -> CalculationResult: characteristics = data["characteristics"] sensors = data["sensors"] return CalculationResult( - schema_version=str(data.get("schemaVersion", "0.2.0-placeholder")), + schema_version=str(data.get("schemaVersion", "0.2.0")), name=str(data["name"]), generated_utc=str(data["generatedUtc"]), characteristics=CharacteristicsResult( diff --git a/src/acoustic_dataset/generate.py b/src/acoustic_dataset/generate.py index 89a80ae..db7bd65 100644 --- a/src/acoustic_dataset/generate.py +++ b/src/acoustic_dataset/generate.py @@ -1,8 +1,8 @@ """Generate typed dataclass models from the XSD via xsdata. This is the *only* code that turns the schema into Python. It is schema-agnostic: it -hard-codes no element names, so swapping in the real XSD (contracts/pipeline-contract.md) -needs no edits here — just re-run ``make generate``. +hard-codes no element names, so changing the schema needs no edits here — just re-run +``make generate``. Two deliberate post-processing steps make the output fit our constraints: diff --git a/src/acoustic_dataset/mapping.py b/src/acoustic_dataset/mapping.py index 254da4a..1cd6358 100644 --- a/src/acoustic_dataset/mapping.py +++ b/src/acoustic_dataset/mapping.py @@ -1,7 +1,7 @@ """The single mapping: ``CalculationResult`` -> populated generated models. This is the **one place schema-aware logic lives** (contracts/pipeline-contract.md). It is -expected to change when the real XSD lands; nothing else references schema element names. +the only place to touch when the schema changes; nothing else references schema element names. Two rules it enforces: diff --git a/tests/golden/acoustic_dataset.xml b/tests/golden/acoustic_dataset.xml index 16716da..6b1204d 100644 --- a/tests/golden/acoustic_dataset.xml +++ b/tests/golden/acoustic_dataset.xml @@ -1,7 +1,7 @@ - 0.2.0-placeholder - Illustrative platform (placeholder) + 0.2.0 + Reference Platform A 2026-06-14T00:00:00Z 7.500 diff --git a/tests/integration/test_pipeline.py b/tests/integration/test_pipeline.py index 232a434..b781376 100644 --- a/tests/integration/test_pipeline.py +++ b/tests/integration/test_pipeline.py @@ -23,7 +23,7 @@ def _build_xml(input_path): def test_populated_objects_carry_the_expected_typed_values(input_path): model = to_model(acoustics.calculate_from_file(input_path)) - assert model.schema_version == "0.2.0-placeholder" + assert model.schema_version == "0.2.0" # Platform characteristics: Decimals (the typed boundary the schema declares), not floats. assert model.characteristics.draft == Decimal("7.500") diff --git a/tests/unit/test_acoustics.py b/tests/unit/test_acoustics.py index 1007306..8f2ba1c 100644 --- a/tests/unit/test_acoustics.py +++ b/tests/unit/test_acoustics.py @@ -60,7 +60,7 @@ def test_calculate_produces_ten_bands_of_twelve_sectors(input_path): def test_calculate_synthesises_characteristics_and_sensors(input_path): result = acoustics.calculate_from_file(input_path) - assert result.schema_version == "0.2.0-placeholder" + assert result.schema_version == "0.2.0" assert result.characteristics.year_introduced == 1998 assert result.active_sonar.name == "AS-900 Echo" assert result.active_sonar.max_range_m == pytest.approx(10.0 ** (203.0 / 40.0)) From da380f0d0c378f85887c31f210f96a166ca14e56 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 14 Jun 2026 13:03:10 +0000 Subject: [PATCH 5/7] docs(schema): add worked examples to the generated schema reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Show, on the schema reference page (all generated from the canonical pipeline run so they can't drift): - Example document — a trimmed, validated XML excerpt of the output. - Working with the typed objects — a Python snippet (load -> to_model -> field reads) with the real values pulled from the populated Platform. - Worked example: deriving a value from elementary physics — the active sonar's MaxRange is computed from SourceLevel and DetectionThreshold via the sonar equation under two-way spherical spreading (r = 10**((SL-DT)/40) = 118850.223 m), and that typed value is exactly what serialises into — typed data in, schema-valid XML out. - schema_docs.generate() gains an optional example_input; `gen-schema-docs` passes --input. - Tests cover the three new sections; the committed-page freshness test still gates drift. Verified on real CPython 3.9.4: ruff + mypy + pytest (44) + mkdocs --strict green; idempotent. --- docs/reference/schema/index.md | 100 +++++++++++++++++++++++ src/acoustic_dataset/cli.py | 5 +- src/acoustic_dataset/schema_docs.py | 111 +++++++++++++++++++++++++- tests/integration/test_schema_docs.py | 75 ++++++++++++----- 4 files changed, 266 insertions(+), 25 deletions(-) diff --git a/docs/reference/schema/index.md b/docs/reference/schema/index.md index a97e452..dfdb6e1 100644 --- a/docs/reference/schema/index.md +++ b/docs/reference/schema/index.md @@ -184,3 +184,103 @@ The numeric primitives below carry real XSD range facets, so an out-of-band valu | `Bearing` | `xs:decimal` | ≥ 0, < 360 | A compass bearing in degrees, in the half-open interval [0, 360): 0 is dead ahead, increasing clockwise. Radiated-noise sectors are spaced 30 degrees apart. | | `Year` | `xs:integer` | ≥ 1900, ≤ 2100 | A Gregorian calendar year, constrained to a sane modern range [1900, 2100]. | +## Example document + +A validated document produced by `make pipeline` from `examples/calculation_input.json` (most radiated-noise detail elided): + +```xml + + 0.2.0 + Reference Platform A + 2026-06-14T00:00:00Z + + 7.500 + 95.000 + 2400.000 + 1998 + + + + 50.000 + + + 0.000 + 134.000 + + + 30.000 + 134.804 + + + + + + + + + AS-900 Echo + DeepBlue Sonics + 6000.000 + 215.000 + 15.000 + 0.500 + 118850.223 + + + PA-110 Flank Array + DeepBlue Sonics + 1500.000 + 18.000 + 10.000 + 1.500 + + + PA-220 Towed Array + Marine Acoustics Ltd + 300.000 + 22.000 + 8.000 + 2.000 + + + +``` + +## Working with the typed objects + +The pipeline maps calculation output **once** onto the generated dataclasses; tests assert on those typed objects (the testable boundary) and they serialise straight to XML: + +```python +from acoustic_dataset import acoustics, serialize +from acoustic_dataset.mapping import to_model + +result = acoustics.calculate_from_file("examples/calculation_input.json") +platform = to_model(result) # a generated Platform object + +platform.name # 'Reference Platform A' +platform.characteristics.draft # Decimal('7.500') +len(platform.radiated_noise.band) # 10 +platform.sensors.active.max_range # Decimal('118850.223') + +xml = serialize.to_xml(platform) # -> the validated document shown above +``` + +## Worked example: deriving a value from elementary physics + +Not every element is copied from the input — some are **computed** from typed inputs. The active sonar's maximum echo range is one: it falls out of the sonar equation under two-way spherical spreading. + +An echo travels out *and back*, so transmission loss is `TL = 40 * log10(r)` dB at range `r` metres. The platform can just detect the returning echo when its source level, less that loss, reaches the detection threshold — solve for `r`: + +```text +SL - 40*log10(r) = DT => r = 10 ** ((SL - DT) / 40) +``` + +This platform's active sonar transmits at `SL = 215` dB with a detection threshold `DT = 12` dB, so: + +```python +from acoustic_dataset.acoustics import active_max_range_m +active_max_range_m(215, 12) # => 118850.223 (metres) +``` + +That typed result is exactly what serialises into the document as `118850.223` — elementary physics over typed inputs, schema-valid XML out. + diff --git a/src/acoustic_dataset/cli.py b/src/acoustic_dataset/cli.py index 5f1140e..0635c0e 100644 --- a/src/acoustic_dataset/cli.py +++ b/src/acoustic_dataset/cli.py @@ -77,10 +77,10 @@ def cmd_validate(args: argparse.Namespace) -> int: def cmd_gen_schema_docs(args: argparse.Namespace) -> int: - """Generate the schema reference + Mermaid ERD from the enriched XSD.""" + """Generate the schema reference + Mermaid ERD (and worked examples) from the schema.""" from acoustic_dataset import schema_docs - out_file = schema_docs.generate(args.schema, args.out) + out_file = schema_docs.generate(args.schema, args.out, example_input=args.input) print(f"schema docs ok: generated {out_file} from {args.schema}") return 0 @@ -158,6 +158,7 @@ def build_parser() -> argparse.ArgumentParser: "gen-schema-docs", help="Generate schema reference + Mermaid ERD from the XSD (US5)." ) p_doc.add_argument("--schema", type=Path, default=DEFAULT_SCHEMA) + p_doc.add_argument("--input", type=Path, default=DEFAULT_INPUT) p_doc.add_argument("--out", type=Path, default=_REPO_ROOT / "docs" / "reference" / "schema") p_doc.set_defaults(func=cmd_gen_schema_docs) diff --git a/src/acoustic_dataset/schema_docs.py b/src/acoustic_dataset/schema_docs.py index c9b429a..f956d6f 100644 --- a/src/acoustic_dataset/schema_docs.py +++ b/src/acoustic_dataset/schema_docs.py @@ -319,10 +319,115 @@ def render_markdown(model: SchemaModel) -> str: # --------------------------------------------------------------------------- entry point -def generate(schema_path: Path, out_dir: Path) -> Path: - """Write the generated schema reference to ``/index.md`` and return its path.""" +def _render_examples(example_input: Path) -> list[str]: + """Worked examples generated from the canonical pipeline run, so they cannot drift: + + a trimmed sample document, typed-object usage, and a field *derived* from elementary physics. + """ + from acoustic_dataset import acoustics, serialize + from acoustic_dataset.mapping import to_model + + raw = acoustics.load_input(example_input) + platform = to_model(acoustics.calculate(raw)) + xml = serialize.to_xml(platform) + + characteristics = platform.characteristics + radiated = platform.radiated_noise + sensors = platform.sensors + assert characteristics is not None and radiated is not None and sensors is not None + active = sensors.active + assert active is not None and active.source_level is not None and active.max_range is not None + + # Trim the document to a compact but representative excerpt (the bulk is radiated noise). + ns = serialize.NAMESPACE + root = etree.fromstring(xml.encode("utf-8"), etree.XMLParser(remove_blank_text=True)) + rn = root.find(f"{{{ns}}}RadiatedNoise") + if rn is not None: + bands = rn.findall(f"{{{ns}}}Band") + for extra in bands[1:]: + rn.remove(extra) + rn.append(etree.Comment(" bands 2-10 omitted for brevity ")) + directional = bands[0].find(f"{{{ns}}}Directional") if bands else None + if directional is not None: + for extra in directional.findall(f"{{{ns}}}Sector")[2:]: + directional.remove(extra) + directional.append(etree.Comment(" sectors 3-12 omitted for brevity ")) + excerpt = etree.tostring(root, pretty_print=True, encoding="unicode").rstrip() + + sl = float(active.source_level) + dt = float(raw["sensors"]["active"]["detectionThresholdDb"]) + derived = acoustics.active_max_range_m(sl, dt) + + return [ + "## Example document", + "", + "A validated document produced by `make pipeline` from " + "`examples/calculation_input.json` (most radiated-noise detail elided):", + "", + "```xml", + excerpt, + "```", + "", + "## Working with the typed objects", + "", + "The pipeline maps calculation output **once** onto the generated dataclasses; tests " + "assert on those typed objects (the testable boundary) and they serialise straight to XML:", + "", + "```python", + "from acoustic_dataset import acoustics, serialize", + "from acoustic_dataset.mapping import to_model", + "", + 'result = acoustics.calculate_from_file("examples/calculation_input.json")', + "platform = to_model(result) # a generated Platform object", + "", + f"platform.name # {platform.name!r}", + f"platform.characteristics.draft # Decimal('{characteristics.draft}')", + f"len(platform.radiated_noise.band) # {len(radiated.band)}", + f"platform.sensors.active.max_range # Decimal('{active.max_range}')", + "", + "xml = serialize.to_xml(platform) # -> the validated document shown above", + "```", + "", + "## Worked example: deriving a value from elementary physics", + "", + "Not every element is copied from the input — some are **computed** from typed inputs. " + "The active sonar's maximum echo range is one: it falls out of the sonar equation under " + "two-way spherical spreading.", + "", + "An echo travels out *and back*, so transmission loss is `TL = 40 * log10(r)` dB at " + "range `r` metres. The platform can just detect the returning echo when its source level, " + "less that loss, reaches the detection threshold — solve for `r`:", + "", + "```text", + "SL - 40*log10(r) = DT => r = 10 ** ((SL - DT) / 40)", + "```", + "", + f"This platform's active sonar transmits at `SL = {sl:g}` dB with a detection threshold " + f"`DT = {dt:g}` dB, so:", + "", + "```python", + "from acoustic_dataset.acoustics import active_max_range_m", + f"active_max_range_m({sl:g}, {dt:g}) # => {derived:.3f} (metres)", + "```", + "", + f"That typed result is exactly what serialises into the document as " + f"`{active.max_range}` — elementary physics over typed inputs, " + "schema-valid XML out.", + "", + ] + + +def generate(schema_path: Path, out_dir: Path, example_input: Path | None = None) -> Path: + """Write the generated schema reference to ``/index.md`` and return its path. + + When ``example_input`` is given, append worked examples (a sample document, typed-object + usage, and a physics-derived field) computed from that input via the pipeline. + """ model = parse_schema(schema_path) + md = render_markdown(model) + if example_input is not None: + md = md.rstrip("\n") + "\n\n" + "\n".join(_render_examples(example_input)) + "\n" out_dir.mkdir(parents=True, exist_ok=True) out_file = out_dir / "index.md" - out_file.write_text(render_markdown(model), encoding="utf-8") + out_file.write_text(md, encoding="utf-8") return out_file diff --git a/tests/integration/test_schema_docs.py b/tests/integration/test_schema_docs.py index 7ed7f0f..32ae1c8 100644 --- a/tests/integration/test_schema_docs.py +++ b/tests/integration/test_schema_docs.py @@ -2,8 +2,9 @@ The schema reference + ERD are generated from the enriched XSD so they cannot drift from the contract. These tests assert the generator emits the entities, the ``xs:documentation`` prose, -and a Mermaid ``erDiagram`` — and that the committed page is byte-identical to a fresh -regeneration (the property the CI drift gate depends on). +a Mermaid ``erDiagram``, and the worked examples (sample document, typed-object usage, and a +physics-derived field) — and that the committed page is byte-identical to a fresh regeneration +(the property the CI drift gate depends on). """ from __future__ import annotations @@ -11,59 +12,93 @@ from acoustic_dataset import schema_docs -def _generate(base, schema_path) -> str: - return schema_docs.generate(schema_path, base / "schema").read_text(encoding="utf-8") +def _generate(base, schema_path, input_path) -> str: + out = schema_docs.generate(schema_path, base / "schema", example_input=input_path) + return out.read_text(encoding="utf-8") -def test_output_contains_a_mermaid_erdiagram(tmp_path, schema_path): - md = _generate(tmp_path, schema_path) +def test_output_contains_a_mermaid_erdiagram(tmp_path, schema_path, input_path): + md = _generate(tmp_path, schema_path, input_path) assert "```mermaid" in md assert "erDiagram" in md -def test_output_contains_every_complex_type_entity(tmp_path, schema_path): - md = _generate(tmp_path, schema_path) +def test_output_contains_every_complex_type_entity(tmp_path, schema_path, input_path): + md = _generate(tmp_path, schema_path, input_path) model = schema_docs.parse_schema(schema_path) assert model.complex_types, "expected the schema to declare complex types" for ct in model.complex_types: assert f"### {ct.name}" in md, f"missing entity section for {ct.name}" -def test_output_carries_xs_documentation_prose(tmp_path, schema_path): +def test_output_carries_xs_documentation_prose(tmp_path, schema_path, input_path): # Prose that lives only in the XSD's xs:annotation/xs:documentation (FR-022). - md = _generate(tmp_path, schema_path) + md = _generate(tmp_path, schema_path, input_path) assert "Centre bearing of the sector, in degrees [0, 360)." in md assert "Maximum echo detection range, in metres." in md -def test_output_lists_banded_types_with_ranges(tmp_path, schema_path): - md = _generate(tmp_path, schema_path) +def test_output_lists_banded_types_with_ranges(tmp_path, schema_path, input_path): + md = _generate(tmp_path, schema_path, input_path) assert "## Banded numeric types" in md assert "≥ -200" in md and "≤ 300" in md # Decibels facet range assert "< 360" in md # Bearing's maxExclusive -def test_erd_shows_relationships_and_cardinalities(tmp_path, schema_path): - md = _generate(tmp_path, schema_path) +def test_erd_shows_relationships_and_cardinalities(tmp_path, schema_path, input_path): + md = _generate(tmp_path, schema_path, input_path) assert "RADIATED_NOISE ||--|{ RADIATED_BAND" in md # one-to-many (10 bands) assert "PLATFORM_TYPE ||--|| SENSOR_SUITE" in md # one-to-one -def test_inheritance_is_flattened_in_derived_entities(tmp_path, schema_path): - md = _generate(tmp_path, schema_path) +def test_inheritance_is_flattened_in_derived_entities(tmp_path, schema_path, input_path): + md = _generate(tmp_path, schema_path, input_path) assert "Extends [`Sonar`]" in md assert "*(from Sonar)*" in md # inherited fields marked in ActiveSonar/PassiveSonar -def test_generation_is_idempotent(tmp_path, schema_path): - assert _generate(tmp_path / "a", schema_path) == _generate(tmp_path / "b", schema_path) +# --- worked examples (sample document, typed-object usage, physics) ---------------------- -def test_committed_page_matches_a_fresh_regeneration(repo_root, schema_path, tmp_path): +def test_example_document_section_has_a_sample_xml(tmp_path, schema_path, input_path): + md = _generate(tmp_path, schema_path, input_path) + assert "## Example document" in md + assert "```xml" in md + assert "118850.223" in md + assert "omitted for brevity" in md # the excerpt is trimmed + + +def test_typed_objects_section_shows_real_values(tmp_path, schema_path, input_path): + md = _generate(tmp_path, schema_path, input_path) + assert "## Working with the typed objects" in md + assert "to_model(result)" in md + assert "'Reference Platform A'" in md # the real platform name + assert "Decimal('118850.223')" in md + + +def test_physics_section_derives_max_range(tmp_path, schema_path, input_path): + md = _generate(tmp_path, schema_path, input_path) + assert "deriving a value from elementary physics" in md + assert "r = 10 ** ((SL - DT) / 40)" in md + assert "active_max_range_m(215, 12)" in md # the real SL / DT + assert "118850.223" in md # the computed range == the element + + +# --- determinism / drift gate ------------------------------------------------------------ + + +def test_generation_is_idempotent(tmp_path, schema_path, input_path): + assert _generate(tmp_path / "a", schema_path, input_path) == _generate( + tmp_path / "b", schema_path, input_path + ) + + +def test_committed_page_matches_a_fresh_regeneration(repo_root, schema_path, input_path, tmp_path): # The property the CI drift gate relies on: regeneration is byte-identical to the commit. committed = (repo_root / "docs" / "reference" / "schema" / "index.md").read_text( encoding="utf-8" ) - assert _generate(tmp_path, schema_path) == committed, ( + assert _generate(tmp_path, schema_path, input_path) == committed, ( "run `make gen-schema-docs` — the committed schema reference is stale" ) From 5cacbf28772ecf3a1a1db7bd7e41a6a17153c850 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 14 Jun 2026 16:59:54 +0000 Subject: [PATCH 6/7] docs(concepts): add "Typed objects vs. dictionaries" with backing tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A focused explanation (with runnable Python) of why the pipeline maps onto the generated typed objects instead of threading generic dicts through (FR-010, ADR 0002): - dict access is stringly-typed (Any), typos fail only at runtime or silently via .get, units live only in key names, and bad values flow straight through; - typed objects give attribute access + autocomplete, mypy/AttributeError on typos, precise Decimal types with meaning from the XSD docstrings, and MappingError at the single boundary for out-of-band values — before serialisation. - New concept page wired into the nav + onboarding mental-model list. - tests/unit/test_typed_vs_dict.py pins every claim (Decimal value, dict KeyError/silent None, attribute typo -> AttributeError, out-of-band -> MappingError) so the page can't rot. Verified on real CPython 3.9.4: ruff + mypy + pytest (48) + mkdocs --strict green. --- docs/concepts/typed-vs-dicts.md | 79 ++++++++++++++++++++++++++++++++ docs/onboarding.md | 1 + mkdocs.yml | 1 + tests/unit/test_typed_vs_dict.py | 48 +++++++++++++++++++ 4 files changed, 129 insertions(+) create mode 100644 docs/concepts/typed-vs-dicts.md create mode 100644 tests/unit/test_typed_vs_dict.py diff --git a/docs/concepts/typed-vs-dicts.md b/docs/concepts/typed-vs-dicts.md new file mode 100644 index 0000000..b6a22cc --- /dev/null +++ b/docs/concepts/typed-vs-dicts.md @@ -0,0 +1,79 @@ +# Typed objects vs. dictionaries + +> **Explanation** — why the pipeline carries data in typed objects generated from the schema, +> not in generic `dict`s. See [ADR 0002](../decisions/0002-drop-csv-pickle-and-write_xml.md): we +> deliberately dropped the CSV/pickle/dict baton-pass. + +The example input arrives as JSON, which Python loads into nested `dict`s. It would be tempting +to thread those dicts straight through to the XML. We don't — the pipeline maps them **once** +onto the [generated dataclasses](../reference/schema/index.md) and works with those. Here's why. + +## The generic dictionary (what we avoid) + +```python +from acoustic_dataset import acoustics + +raw = acoustics.load_input("examples/calculation_input.json") # nested dicts + +raw["sensors"]["active"]["sourceLevelDb"] # 215.0 — a float; the unit lives only in the key +raw["snesors"] # KeyError — a typo anywhere fails only at runtime +raw["sensors"]["active"].get("srcLevel") # None — or fails *silently*, with no error at all +``` + +A `dict` is an opaque bag of `Any`: + +- **No structure** — every access is a stringly-typed guess; the editor can't autocomplete and a + typo (`snesors`, `srcLevel`) isn't caught until that line runs — or never, if you used `.get`. +- **No types** — values are `float`/`str`/`Any`; nothing says `sourceLevelDb` is decibels, and a + bare `7.5` could be metres, feet, or a mistake. +- **No validation** — a nonsense value (a 9999 dB source level) flows straight through; nothing + stops it becoming schema-invalid XML downstream. + +## The typed object (what the pipeline uses) + +```python +from acoustic_dataset import acoustics, serialize +from acoustic_dataset.mapping import to_model + +platform = to_model(acoustics.calculate_from_file("examples/calculation_input.json")) + +platform.sensors.active.source_level # Decimal('215.000') — the exact type the schema declares +platform.sensors.active.sourceLevel # AttributeError + a mypy error — the typo is caught +xml = serialize.to_xml(platform) # the object graph *is* the document +``` + +The generated dataclasses encode the contract: + +- **Typos are caught** — statically by `mypy` and at runtime by `AttributeError`, never silently. +- **Precise types** — `Decimal` (exact, not a lossy `float`); the field name plus its docstring + (lifted from the XSD's `xs:documentation`) carry the meaning, and the + [schema reference](../reference/schema/index.md) documents the whole shape. +- **Validated at one boundary** — the single mapping rejects anything outside the schema's bands + *before* serialisation: + +```python +import dataclasses +from acoustic_dataset.mapping import to_model, MappingError + +result = acoustics.calculate_from_file("examples/calculation_input.json") +# the schema bounds Decibels to [-200, 300]; force an impossible source level: +bad = dataclasses.replace( + result, active_sonar=dataclasses.replace(result.active_sonar, source_level_db=9999.0) +) +to_model(bad) # raises MappingError — a dict would have passed 9999 straight through +``` + +## The pay-off + +| | `dict` | Typed object | +|---|---|---| +| Find a field | guess a string key | attribute + autocomplete | +| Wrong name | `KeyError` / silent `None`, at runtime | `mypy` error + `AttributeError` | +| Value type | `Any` / `float` | `Decimal`, schema-banded | +| Units / meaning | only in the key name | field name + generated docstring | +| Bad value | flows through | `MappingError` at the boundary | +| Is it the contract? | no — an ad-hoc shape | yes — generated from the XSD | + +This is the **typed, testable boundary** the pipeline is built around (FR-010): tests assert on +these objects directly, and they serialise straight to validated XML. Every behaviour shown here +is checked in `tests/unit/test_typed_vs_dict.py`, so the comparison can't quietly rot. diff --git a/docs/onboarding.md b/docs/onboarding.md index 48fab58..5a69d2f 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -41,6 +41,7 @@ Read these, in order — they're short: 1. [Schema as the contract](concepts/schema-as-contract.md) — the idea everything follows from. 2. [The two verification gates](concepts/two-verification-gates.md) — why schema-valid isn't the same as correct. 3. [Pipeline data flow](concepts/pipeline-data-flow.md) — how input becomes validated XML. +4. [Typed objects vs dictionaries](concepts/typed-vs-dicts.md) — why typed data beats a generic `dict`. !!! tip "What's done, what's next" The full Phase 1 pipeline is in place — environment, end-to-end pipeline, migration-safety diff --git a/mkdocs.yml b/mkdocs.yml index fa19595..ccfe65c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -54,6 +54,7 @@ nav: - Schema as the contract: concepts/schema-as-contract.md - The two verification gates: concepts/two-verification-gates.md - Pipeline data flow (ERD): concepts/pipeline-data-flow.md + - Typed objects vs dictionaries: concepts/typed-vs-dicts.md - Decisions (ADRs): - Overview: decisions/index.md - ADR template: decisions/0000-template.md diff --git a/tests/unit/test_typed_vs_dict.py b/tests/unit/test_typed_vs_dict.py new file mode 100644 index 0000000..60d1d04 --- /dev/null +++ b/tests/unit/test_typed_vs_dict.py @@ -0,0 +1,48 @@ +"""Back the 'Typed objects vs. dictionaries' concept page with executable proof. + +Each test pins one claim the page makes, so the comparison can't quietly rot (FR-010, ADR 0002). +""" + +from __future__ import annotations + +import dataclasses +from decimal import Decimal + +import pytest + +from acoustic_dataset import acoustics +from acoustic_dataset.mapping import MappingError, to_model + + +def test_typed_value_is_a_precise_decimal(input_path): + platform = to_model(acoustics.calculate_from_file(input_path)) + source_level = platform.sensors.active.source_level + assert isinstance(source_level, Decimal) # not a lossy float + assert source_level == Decimal("215.000") # the exact, schema-banded value + + +def test_dict_access_is_stringly_typed_and_unsafe(input_path): + raw = acoustics.load_input(input_path) + # A real value — but keyed by an ad-hoc string and typed only as float. + assert raw["sensors"]["active"]["sourceLevelDb"] == 215.0 + # A typo in the path explodes only at runtime... + with pytest.raises(KeyError): + _ = raw["snesors"] + # ...or, with .get, silently returns None — wrong, with no error at all. + assert raw["sensors"]["active"].get("srcLevel") is None + + +def test_typed_attribute_typo_is_caught(input_path): + platform = to_model(acoustics.calculate_from_file(input_path)) + with pytest.raises(AttributeError): + _ = platform.sensors.active.sourceLevel # not a field — mypy + runtime catch it + + +def test_out_of_band_value_is_rejected_at_the_typed_boundary(input_path): + result = acoustics.calculate_from_file(input_path) + # The schema bounds Decibels to [-200, 300]; a dict would carry 9999 straight through. + bad = dataclasses.replace( + result, active_sonar=dataclasses.replace(result.active_sonar, source_level_db=9999.0) + ) + with pytest.raises(MappingError, match="SourceLevel"): + to_model(bad) From 4d9cc90cee8314d357ea80506c034dbb5d02448e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 14 Jun 2026 21:44:55 +0000 Subject: [PATCH 7/7] docs(concepts): neutral, storage-focused rewrite of typed-vs-dicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback: educational/neutral tone (not persuasive), and focus on what strong typing constrains when data is *stored* rather than when it is read back. - typed-vs-dicts.md: rewritten around storing data — a dict accepts any key/type/range; a typed object fixes the field set (unknown name -> TypeError at construction), declares each field's type (mypy catches wrong types), and the mapping rejects out-of-range values as they are stored. Plain summary table; removed the salesy phrasing. - test_typed_vs_dict.py: aligned to verified storage behaviour (dict stores anything; unknown field -> TypeError; stored field is Decimal; out-of-range -> MappingError). Dropped the read-time attribute-typo example. - schema_docs.py: neutralise the one punchy line in the generated worked example; regenerate. Verified on real CPython 3.9.4: ruff + mypy + pytest (48) + mkdocs --strict green; idempotent. --- docs/concepts/typed-vs-dicts.md | 99 ++++++++++++++--------------- docs/reference/schema/index.md | 2 +- src/acoustic_dataset/schema_docs.py | 5 +- tests/unit/test_typed_vs_dict.py | 38 +++++------ 4 files changed, 71 insertions(+), 73 deletions(-) diff --git a/docs/concepts/typed-vs-dicts.md b/docs/concepts/typed-vs-dicts.md index b6a22cc..725a9ff 100644 --- a/docs/concepts/typed-vs-dicts.md +++ b/docs/concepts/typed-vs-dicts.md @@ -1,79 +1,78 @@ # Typed objects vs. dictionaries -> **Explanation** — why the pipeline carries data in typed objects generated from the schema, -> not in generic `dict`s. See [ADR 0002](../decisions/0002-drop-csv-pickle-and-write_xml.md): we -> deliberately dropped the CSV/pickle/dict baton-pass. +> **Explanation** — the input is loaded as JSON (nested `dict`s), but the pipeline maps it once +> onto typed objects generated from the schema and stores data in those. This page sets out what +> strong typing constrains at the point data is stored. See +> [ADR 0002](../decisions/0002-drop-csv-pickle-and-write_xml.md). -The example input arrives as JSON, which Python loads into nested `dict`s. It would be tempting -to thread those dicts straight through to the XML. We don't — the pipeline maps them **once** -onto the [generated dataclasses](../reference/schema/index.md) and works with those. Here's why. +## Storing data in a dictionary -## The generic dictionary (what we avoid) +A `dict` places no constraints on what it holds: keys are arbitrary strings and values are `Any`. ```python -from acoustic_dataset import acoustics - -raw = acoustics.load_input("examples/calculation_input.json") # nested dicts - -raw["sensors"]["active"]["sourceLevelDb"] # 215.0 — a float; the unit lives only in the key -raw["snesors"] # KeyError — a typo anywhere fails only at runtime -raw["sensors"]["active"].get("srcLevel") # None — or fails *silently*, with no error at all +record = {} +record["sourceLevel"] = 215.0 # any key, any value type +record["sorceLevel"] = 9999 # a misspelled key is just another entry +record["sourceLevel"] = "loud" # a string replaces the number, with no objection ``` -A `dict` is an opaque bag of `Any`: +The structure exists only by convention. A misspelled key, a wrong value type, or an omitted +field is stored as readily as correct data, so a mistake surfaces later — when something reads +the value, when the XML fails validation, or not at all. -- **No structure** — every access is a stringly-typed guess; the editor can't autocomplete and a - typo (`snesors`, `srcLevel`) isn't caught until that line runs — or never, if you used `.get`. -- **No types** — values are `float`/`str`/`Any`; nothing says `sourceLevelDb` is decibels, and a - bare `7.5` could be metres, feet, or a mistake. -- **No validation** — a nonsense value (a 9999 dB source level) flows straight through; nothing - stops it becoming schema-invalid XML downstream. +## Storing data in a typed object -## The typed object (what the pipeline uses) +The generated dataclasses declare which fields exist and the type of each, so the shape of what +is stored is defined up front: ```python -from acoustic_dataset import acoustics, serialize -from acoustic_dataset.mapping import to_model +from decimal import Decimal +from acoustic_dataset.models.acoustic_dataset import Sector -platform = to_model(acoustics.calculate_from_file("examples/calculation_input.json")) - -platform.sensors.active.source_level # Decimal('215.000') — the exact type the schema declares -platform.sensors.active.sourceLevel # AttributeError + a mypy error — the typo is caught -xml = serialize.to_xml(platform) # the object graph *is* the document +Sector(bearing=Decimal("30.000"), level=Decimal("134.000")) # the declared fields +Sector(bering=Decimal("30.000"), level=Decimal("134.000")) # TypeError: unexpected 'bering' ``` -The generated dataclasses encode the contract: +- A name that is not a declared field is rejected when the object is constructed (`TypeError`), + where a `dict` would store it under a new key. +- Each field has a declared type — here `Decimal`, as the schema specifies. A type checker + (`mypy`, run by `make verify`) reports a wrong-typed value before the code runs: + + ```python + Sector(bearing="thirty", level=Decimal("134.000")) # mypy: incompatible type "str" + ``` -- **Typos are caught** — statically by `mypy` and at runtime by `AttributeError`, never silently. -- **Precise types** — `Decimal` (exact, not a lossy `float`); the field name plus its docstring - (lifted from the XSD's `xs:documentation`) carry the meaning, and the - [schema reference](../reference/schema/index.md) documents the whole shape. -- **Validated at one boundary** — the single mapping rejects anything outside the schema's bands - *before* serialisation: +- The fields and their documentation are generated from the schema, so the stored object follows + the contract rather than an ad-hoc shape. + +## Checking values as they are stored + +Field names and types define the *shape*. The single mapping (`to_model`) is where calculation +output is stored into these objects, and it also enforces the schema's numeric **ranges** at that +point: ```python import dataclasses +from acoustic_dataset import acoustics from acoustic_dataset.mapping import to_model, MappingError result = acoustics.calculate_from_file("examples/calculation_input.json") -# the schema bounds Decibels to [-200, 300]; force an impossible source level: +# Decibels are bounded to [-200, 300]; attempt to store an impossible source level: bad = dataclasses.replace( result, active_sonar=dataclasses.replace(result.active_sonar, source_level_db=9999.0) ) -to_model(bad) # raises MappingError — a dict would have passed 9999 straight through +to_model(bad) # MappingError — rejected as it is stored, not left for a later stage ``` -## The pay-off +A `dict` would hold `9999` and pass it on; the typed boundary rejects it. -| | `dict` | Typed object | +## Summary + +| At the point data is stored | `dict` | Typed object | |---|---|---| -| Find a field | guess a string key | attribute + autocomplete | -| Wrong name | `KeyError` / silent `None`, at runtime | `mypy` error + `AttributeError` | -| Value type | `Any` / `float` | `Decimal`, schema-banded | -| Units / meaning | only in the key name | field name + generated docstring | -| Bad value | flows through | `MappingError` at the boundary | -| Is it the contract? | no — an ad-hoc shape | yes — generated from the XSD | - -This is the **typed, testable boundary** the pipeline is built around (FR-010): tests assert on -these objects directly, and they serialise straight to validated XML. Every behaviour shown here -is checked in `tests/unit/test_typed_vs_dict.py`, so the comparison can't quietly rot. +| Field names | any string accepted | declared; an unknown name is a `TypeError` | +| Value types | `Any` | declared (e.g. `Decimal`), checked by `mypy` | +| Out-of-range values | stored as-is | rejected by the mapping (`MappingError`) | +| Relationship to the schema | convention only | generated from it | + +The behaviours above are checked in `tests/unit/test_typed_vs_dict.py`. diff --git a/docs/reference/schema/index.md b/docs/reference/schema/index.md index dfdb6e1..b1e9eef 100644 --- a/docs/reference/schema/index.md +++ b/docs/reference/schema/index.md @@ -282,5 +282,5 @@ from acoustic_dataset.acoustics import active_max_range_m active_max_range_m(215, 12) # => 118850.223 (metres) ``` -That typed result is exactly what serialises into the document as `118850.223` — elementary physics over typed inputs, schema-valid XML out. +That value serialises into the document as `118850.223`. diff --git a/src/acoustic_dataset/schema_docs.py b/src/acoustic_dataset/schema_docs.py index f956d6f..d25d3b7 100644 --- a/src/acoustic_dataset/schema_docs.py +++ b/src/acoustic_dataset/schema_docs.py @@ -410,9 +410,8 @@ def _render_examples(example_input: Path) -> list[str]: f"active_max_range_m({sl:g}, {dt:g}) # => {derived:.3f} (metres)", "```", "", - f"That typed result is exactly what serialises into the document as " - f"`{active.max_range}` — elementary physics over typed inputs, " - "schema-valid XML out.", + f"That value serialises into the document as " + f"`{active.max_range}`.", "", ] diff --git a/tests/unit/test_typed_vs_dict.py b/tests/unit/test_typed_vs_dict.py index 60d1d04..6ce726c 100644 --- a/tests/unit/test_typed_vs_dict.py +++ b/tests/unit/test_typed_vs_dict.py @@ -1,6 +1,7 @@ """Back the 'Typed objects vs. dictionaries' concept page with executable proof. -Each test pins one claim the page makes, so the comparison can't quietly rot (FR-010, ADR 0002). +Each test pins one behaviour the page describes about storing data, so the comparison stays +accurate (FR-010, ADR 0002). """ from __future__ import annotations @@ -12,33 +13,32 @@ from acoustic_dataset import acoustics from acoustic_dataset.mapping import MappingError, to_model +from acoustic_dataset.models.acoustic_dataset import Sector -def test_typed_value_is_a_precise_decimal(input_path): - platform = to_model(acoustics.calculate_from_file(input_path)) - source_level = platform.sensors.active.source_level - assert isinstance(source_level, Decimal) # not a lossy float - assert source_level == Decimal("215.000") # the exact, schema-banded value +def test_a_dict_stores_anything_silently(): + record: dict = {} + record["sourceLevel"] = 215.0 + record["sorceLevel"] = 9999 # a misspelled key is just another entry + record["sourceLevel"] = "loud" # a wrong type replaces the value, no error + assert record == {"sourceLevel": "loud", "sorceLevel": 9999} -def test_dict_access_is_stringly_typed_and_unsafe(input_path): - raw = acoustics.load_input(input_path) - # A real value — but keyed by an ad-hoc string and typed only as float. - assert raw["sensors"]["active"]["sourceLevelDb"] == 215.0 - # A typo in the path explodes only at runtime... - with pytest.raises(KeyError): - _ = raw["snesors"] - # ...or, with .get, silently returns None — wrong, with no error at all. - assert raw["sensors"]["active"].get("srcLevel") is None +def test_an_unknown_field_is_rejected_when_constructing(): + # The declared shape is accepted... + Sector(bearing=Decimal("30.000"), level=Decimal("134.000")) + # ...but a name that is not a field fails at construction (a dict would just store it). + with pytest.raises(TypeError): + Sector(bering=Decimal("30.000"), level=Decimal("134.000")) -def test_typed_attribute_typo_is_caught(input_path): +def test_a_stored_field_has_the_schema_declared_type(input_path): platform = to_model(acoustics.calculate_from_file(input_path)) - with pytest.raises(AttributeError): - _ = platform.sensors.active.sourceLevel # not a field — mypy + runtime catch it + level = platform.radiated_noise.band[0].directional.sector[0].level + assert isinstance(level, Decimal) # stored as the schema's Decimal, not a bare float -def test_out_of_band_value_is_rejected_at_the_typed_boundary(input_path): +def test_out_of_range_value_is_rejected_as_it_is_stored(input_path): result = acoustics.calculate_from_file(input_path) # The schema bounds Decibels to [-200, 300]; a dict would carry 9999 straight through. bad = dataclasses.replace(