From 62fb1007f51d328a4db81aa5526cb7eb894feeaa Mon Sep 17 00:00:00 2001 From: vishal-agami Date: Sat, 25 Jul 2026 11:32:26 +0530 Subject: [PATCH 1/6] refactor(core): rename per-datasource org.yaml/ORGANIZATION.md -> datasource.yaml/datasource.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The company-wide organization.yaml (F15) made the per-datasource files' "org" naming ambiguous. Rename them to say what they are — one datasource: - /org.yaml -> /datasource.yaml - /ORGANIZATION.md -> /datasource.md - model root field organization: -> datasource: (pydantic Organization.datasource) - served memory row kind='organization' -> kind='datasource' The company record (organization.yaml, org_id, tenancy), the MCP output field DatasourceSchemaResult.organization, the pydantic class name Organization, and the model-explorer "Organization" tab are intentionally unchanged. The vendored plugins/agami/lib/ slice is regenerated via `dev.py sync-lib`. No on-disk migration: existing profiles must re-run agami-connect or rename the two files by hand (documented in CHANGELOG). Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 14 ++++++- dev/reset-yamls.sh | 16 ++++---- docs/format-spec.md | 14 +++---- docs/mcp-server.md | 2 +- docs/privacy.md | 4 +- docs/troubleshooting.md | 2 +- docs/usage.md | 4 +- packages/agami-core/src/admin.py | 10 ++--- packages/agami-core/src/agami_paths.py | 2 +- packages/agami-core/src/execute_sql.py | 2 +- packages/agami-core/src/mcp_http.py | 6 +-- .../src/migrations/core/001_serving.sql | 6 +-- packages/agami-core/src/model_deploy.py | 24 +++++------ packages/agami-core/src/model_store.py | 22 +++++----- .../agami-core/src/semantic_model/__init__.py | 2 +- .../agami-core/src/semantic_model/build.py | 10 ++--- packages/agami-core/src/semantic_model/cli.py | 20 +++++----- .../agami-core/src/semantic_model/curate.py | 14 +++---- .../src/semantic_model/introspect.py | 4 +- .../agami-core/src/semantic_model/loader.py | 18 ++++----- .../agami-core/src/semantic_model/models.py | 12 +++--- .../src/semantic_model/org_draft.py | 20 +++++----- .../src/semantic_model/org_record.py | 16 ++++---- .../agami-core/src/semantic_model/snapshot.py | 2 +- packages/agami-core/src/tools.py | 30 +++++++------- packages/agami-core/src/ui.py | 2 +- plugins/agami/lib/agami_paths.py | 2 +- plugins/agami/lib/execute_sql.py | 2 +- plugins/agami/lib/semantic_model/__init__.py | 2 +- .../model/{ORGANIZATION.md => datasource.md} | 0 .../store/model/{org.yaml => datasource.yaml} | 2 +- plugins/agami/scripts/parse_model_feedback.py | 2 +- plugins/agami/scripts/prepare_deploy.py | 10 ++--- .../agami/scripts/render_model_explorer.py | 4 +- plugins/agami/shared/credentials-format.md | 2 +- .../shared/examples-validation-template.html | 4 +- plugins/agami/shared/file-layout.md | 10 ++--- .../agami/shared/model-explorer-template.html | 22 +++++----- .../shared/organization-context-format.md | 24 +++++------ plugins/agami/shared/user-memory-format.md | 18 ++++----- plugins/agami/skills/agami-connect/SKILL.md | 40 +++++++++---------- plugins/agami/skills/agami-deploy/SKILL.md | 4 +- plugins/agami/skills/agami-model/SKILL.md | 14 +++---- plugins/agami/skills/agami-query/SKILL.md | 14 +++---- plugins/agami/skills/agami-reconcile/SKILL.md | 2 +- .../skills/agami-save-correction/SKILL.md | 34 ++++++++-------- plugins/agami/skills/agami-serve/SKILL.md | 2 +- tests/test_ace046_incremental_curation.py | 8 ++-- tests/test_ace047_schema_index.py | 8 ++-- tests/test_ace051_fail_closed.py | 6 +-- tests/test_admin_model.py | 12 +++--- tests/test_aggregation_enforcement.py | 4 +- tests/test_cloud_neutral_config.py | 2 +- tests/test_column_scope_adversarial.py | 2 +- tests/test_column_scope_gate.py | 4 +- tests/test_compose_org_context.py | 2 +- tests/test_coverage_gate_self_evident.py | 2 +- tests/test_coverage_underenriched.py | 2 +- tests/test_derived_metrics.py | 2 +- tests/test_entity_anchor_and_area_desc.py | 4 +- tests/test_get_datasource_schema_sizing.py | 4 +- tests/test_guard_context.py | 2 +- tests/test_mcp_harness.py | 10 ++--- tests/test_metadata_sources.py | 14 +++---- tests/test_metric_binding_and_prune.py | 2 +- tests/test_model_deploy.py | 24 +++++------ tests/test_model_snapshots.py | 10 ++--- tests/test_model_store_roundtrip.py | 16 ++++---- tests/test_multi_area_membership.py | 2 +- tests/test_org_cache.py | 8 ++-- tests/test_org_draft.py | 12 +++--- tests/test_org_identity_e2e.py | 12 +++--- tests/test_org_record.py | 14 +++---- tests/test_org_record_deploy.py | 6 +-- tests/test_organization_context_e2e.py | 4 +- tests/test_prepare_deploy.py | 20 +++++----- tests/test_render_model_explorer.py | 10 ++--- tests/test_second_order_metrics.py | 2 +- tests/test_semantic_model_cli.py | 8 ++-- tests/test_semantic_model_curate.py | 18 ++++----- tests/test_semantic_model_introspect.py | 4 +- tests/test_semantic_model_loader.py | 8 ++-- tests/test_semantic_model_models.py | 4 +- tests/test_semantic_model_runtime.py | 6 +-- tests/test_semantic_model_validator_v2.py | 8 ++-- tests/test_sensitive_gate.py | 4 +- tests/test_table_scope_gate.py | 4 +- tests/test_units.py | 12 +++--- 88 files changed, 405 insertions(+), 393 deletions(-) rename plugins/agami/samples/store/model/{ORGANIZATION.md => datasource.md} (100%) rename plugins/agami/samples/store/model/{org.yaml => datasource.yaml} (93%) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf5fae60..d3a9ee3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,18 @@ is the source of truth a host installs against — bumping it is what invalidate user's plugin cache (see [CONTRIBUTING.md](CONTRIBUTING.md)). Each released section below corresponds to one such version. +## [Unreleased] + +### Changed + +- **Renamed the per-datasource model files** so their names say what they are, now that a + company-wide `organization.yaml` exists at the artifacts root. In each profile, + `org.yaml` → `datasource.yaml` and `ORGANIZATION.md` → `datasource.md`. The model root's + display-name field `organization:` is likewise `datasource:`, and the served per-datasource + memory row uses `kind='datasource'`. The company record (`organization.yaml`, `org_id`, + tenancy) is unchanged. No on-disk migration is provided — re-run `agami-connect` (or rename + the two files by hand) for any pre-existing profile. + ## [0.4.5] — 2026-07-15 Hosted/self-hosted server hardening: a real-wheel packaging fix, a multi-tenancy seam, and an @@ -293,7 +305,7 @@ result size, and concurrent load grow. Behaviour-preserving unless a note says o - **Self-host deploy no longer crash-loops on artifact permissions.** The team server runs as a non-root container user; the deploy now stages the model **world-readable**, so the boot-time model load can't fail `Permission denied` - on `ORGANIZATION.md` under a mismatched host owner. + on `datasource.md` under a mismatched host owner. - **claude.ai connects to a self-hosted server.** The `/mcp` endpoint no longer answers the bare (no-trailing-slash) URL with a `307` redirect that the MCP client won't follow — the server normalizes it internally, so `{base}/mcp` diff --git a/dev/reset-yamls.sh b/dev/reset-yamls.sh index 2af038c2..7a69bb8e 100755 --- a/dev/reset-yamls.sh +++ b/dev/reset-yamls.sh @@ -4,7 +4,7 @@ # hand-written docs, or the git audit trail. # # What gets nuked (default): -# //org.yaml (model root) +# //datasource.yaml (model root) # //datasources/ (storage connections) # //subject_areas/ (tables/entities/metrics/relationships) # //prompt_examples/ (NL→SQL example library) @@ -16,7 +16,7 @@ # /local/credentials DB connection details # /local/.config reviewer email, role, etc. # /USER_MEMORY.md cross-DB preferences -# //ORGANIZATION.md domain context the user wrote +# //datasource.md domain context the user wrote # //.git/ audit trail of past curator edits # //.legacy_backup/ legacy (v1) model backed up on upgrade # //curation_log.jsonl @@ -29,7 +29,7 @@ # Usage: # dev/reset-yamls.sh # default profile, soft reset # dev/reset-yamls.sh acme # specific profile, soft reset -# dev/reset-yamls.sh acme --hard # also drop ORGANIZATION.md + .git/ + logs +# dev/reset-yamls.sh acme --hard # also drop datasource.md + .git/ + logs # dev/reset-yamls.sh acme --clean-renders # also drop /local//acme/* (this profile only) # dev/reset-yamls.sh acme --clean-renders-all # wipe legacy flat-layout files + every profile # dev/reset-yamls.sh acme --hard --clean-renders # full per-profile wipe @@ -94,7 +94,7 @@ action() { } echo "Resetting YAMLs in $PROFILE_DIR (profile: $PROFILE)" -echo " mode: $([[ $HARD -eq 1 ]] && echo 'hard (drops ORGANIZATION.md, .git/, logs)' || echo 'soft (keeps ORGANIZATION.md, .git/, logs)')" +echo " mode: $([[ $HARD -eq 1 ]] && echo 'hard (drops datasource.md, .git/, logs)' || echo 'soft (keeps datasource.md, .git/, logs)')" if [[ $CLEAN_RENDERS_ALL -eq 1 ]]; then echo " also cleaning /local/{charts,review,examples-validation,model,exports}/ (ALL profiles + legacy)" elif [[ $CLEAN_RENDERS -eq 1 ]]; then @@ -109,7 +109,7 @@ if [[ -d "$PROFILE_DIR/.snapshots" ]]; then action "chmod -R u+w '$PROFILE_DIR/.snapshots'" fi -# 1. Top-level YAML files (org.yaml, etc.) +# 1. Top-level YAML files (datasource.yaml, etc.) while IFS= read -r f; do [[ -z "$f" ]] && continue action "rm -f '$f'" @@ -132,9 +132,9 @@ if [[ -d "$PROFILE_DIR/.snapshots" ]]; then action "rm -rf '$PROFILE_DIR/.snapshots'" fi -# 4. Hard reset extras: ORGANIZATION.md, the audit trail (.git/), and logs. +# 4. Hard reset extras: datasource.md, the audit trail (.git/), and logs. if [[ $HARD -eq 1 ]]; then - action "rm -f '$PROFILE_DIR/ORGANIZATION.md'" + action "rm -f '$PROFILE_DIR/datasource.md'" action "rm -rf '$PROFILE_DIR/.git'" action "rm -f '$PROFILE_DIR/curation_log.jsonl'" action "rm -f '$PROFILE_DIR/corrections.jsonl'" @@ -186,7 +186,7 @@ fi if [[ $HARD -eq 0 ]]; then echo "" echo "Preserved in soft mode (would be dropped on --hard):" - [[ -f "$PROFILE_DIR/ORGANIZATION.md" ]] && echo " $PROFILE_DIR/ORGANIZATION.md" + [[ -f "$PROFILE_DIR/datasource.md" ]] && echo " $PROFILE_DIR/datasource.md" [[ -d "$PROFILE_DIR/.git" ]] && echo " $PROFILE_DIR/.git (audit trail)" [[ -f "$PROFILE_DIR/curation_log.jsonl" ]] && echo " $PROFILE_DIR/curation_log.jsonl" [[ -f "$PROFILE_DIR/corrections.jsonl" ]] && echo " $PROFILE_DIR/corrections.jsonl" diff --git a/docs/format-spec.md b/docs/format-spec.md index 7ed6628f..aa915205 100644 --- a/docs/format-spec.md +++ b/docs/format-spec.md @@ -23,19 +23,19 @@ agami's state splits across two directories. See [plugins/agami/shared/file-layo | File | Format | Owner | | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | | `/USER_MEMORY.md` | Free-form Markdown — cross-database preferences | Seeded by `agami-connect` Phase 0a, edited by user or appended by `agami-save-correction` | -| `//org.yaml` | Org description + storage-connection refs + subject-area refs + cross_subject_area_relationships | Skill-written, user-editable | +| `//datasource.yaml` | Org description + storage-connection refs + subject-area refs + cross_subject_area_relationships | Skill-written, user-editable | | `//datasources//storage.yaml` | Storage connection (physical): storage_type + storage_config (env-var refs, never secrets) | Skill-written, user-editable | | `//subject_areas//subject_area.yaml` | Subject area: description, default_time_window, TableRefs (with optional expose_column_groups) | Skill-written, user-editable | | `//subject_areas//tables/.yaml` | Canonical Table: columns, grain, default_filters, caveats, column_groups, performance_hints | Skill-written, user-editable | | `//subject_areas//{entities,metrics}/.yaml`, `relationships.yaml` | Entities, metrics, and the intra-area FK graph (join cardinality + trust block) | Skill-written, user-editable | | `//prompt_examples//examples.yaml` | NL→SQL few-shot library (scope-tagged) | Skill-written (seeds) + append-only via `agami-save-correction` | -| `//ORGANIZATION.md` | Free-form Markdown — per-database domain context | Seeded by `agami-connect`, edited by user or appended by `agami-save-correction` | +| `//datasource.md` | Free-form Markdown — per-database domain context | Seeded by `agami-connect`, edited by user or appended by `agami-save-correction` | | `/local/cross_profile_relationships.yaml` | Agami-bespoke YAML — declared JOIN paths across profiles for federation. **Lives in `/local/` because it spans profiles** and isn't tied to one team's repo | User-edited (optional) | `USER_MEMORY.md` is **distinct** from Claude Code's auto-memory at `~/.claude/projects//memory/MEMORY.md`. The auto-memory is host-managed and project-scoped; `USER_MEMORY.md` is agami-managed, lives in the artifacts dir, and persists across Claude Code hosts (CLI / VS Code extension / Cursor extension) the same way the rest of `/` does. -`USER_MEMORY.md` covers **user preferences that apply across every database** (default time windows, display preferences, exclude rules). The per-database **`ORGANIZATION.md`** at `//ORGANIZATION.md` covers **domain knowledge for that specific database** (terminology, key metrics, what the data represents). See [plugins/agami/shared/organization-context-format.md](../plugins/agami/shared/organization-context-format.md). +`USER_MEMORY.md` covers **user preferences that apply across every database** (default time windows, display preferences, exclude rules). The per-database **`datasource.md`** at `//datasource.md` covers **domain knowledge for that specific database** (terminology, key metrics, what the data represents). See [plugins/agami/shared/organization-context-format.md](../plugins/agami/shared/organization-context-format.md). `` matches the section name in `/local/credentials` (default: `default`). One *directory* per profile under `/`. The `agami-connect` skill auto-migrates v1.0 (single-file) and v1.1 (under `/local/`) installs on first run after upgrade. @@ -44,7 +44,7 @@ agami's state splits across two directories. See [plugins/agami/shared/file-layo Three concrete wins (full design in [shared/file-layout.md](../plugins/agami/shared/file-layout.md)): 1. **Zero credential-leak risk on commit.** `/local/` is gitignored by default; `/` is the only place anything goes when teams share. -2. **Team workflows just work.** `cd ~/code/myteam/data && git add agami/` commits everyone's tuned semantic model, examples, ORGANIZATION.md, and USER_MEMORY.md preferences. +2. **Team workflows just work.** `cd ~/code/myteam/data && git add agami/` commits everyone's tuned semantic model, examples, datasource.md, and USER_MEMORY.md preferences. 3. **Power users override per-environment.** Set `AGAMI_ARTIFACTS_DIR=/path/to/staging-models` for an experimental session. --- @@ -70,7 +70,7 @@ password = mypassword The model is a **provider-portable, standard-concepts hierarchy** that any LLM can traverse to build reliable SQL against any backend: ``` -Organization (org.yaml) +Organization (datasource.yaml) ├─ Storage Connections[] (physical: host/creds/dialect — datasources//storage.yaml) └─ Subject Areas[] (logical — the primary unit the LLM consumes; cap ~20-30 tables) ├─ tables[] (TableRefs into storage connections; expose_column_groups scopes wide tables) @@ -88,8 +88,8 @@ Every write is gated by the validator at [`packages/agami-core/src/semantic_mode ### Worked example — a minimal model on disk ```yaml -# org.yaml -organization: shop +# datasource.yaml +datasource: shop version: 1 description: E-commerce shop. storage_connections: diff --git a/docs/mcp-server.md b/docs/mcp-server.md index 6c4d188a..a509ed4c 100644 --- a/docs/mcp-server.md +++ b/docs/mcp-server.md @@ -25,7 +25,7 @@ Four tools, mirroring the hosted connector so the client experience is identical | Tool | What it does | |---|---| | `list_datasources` | Enumerate local profiles (credential sections) and whether each has a model. | -| `get_datasource_schema` | Return the semantic model: the subject-area index, full per-table detail for requested `dataset_names`, plus `ORGANIZATION.md` / `USER_MEMORY.md`. | +| `get_datasource_schema` | Return the semantic model: the subject-area index, full per-table detail for requested `dataset_names`, plus `datasource.md` / `USER_MEMORY.md`. | | `get_prompt_examples` | Return the curated `examples.yaml` few-shot library. | | `execute_sql` | Run **one read-only** `SELECT` / `WITH...SELECT` locally and return `{columns, rows, row_count, ...}`. DML/DDL/multi-statement are rejected. | diff --git a/docs/privacy.md b/docs/privacy.md index 8db50571..a09b0c2b 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -43,9 +43,9 @@ Every byte agami reads or writes stays on your machine: - **Credentials** (`/local/credentials`) — chmod 600 - **Auth files** (`/local/.pgpass`, `.mysql.cnf`, `.snowsql.cnf`) — chmod 600, written by `setup_pgauth.py` - **Config** (`/local/.config`) — `active_profile`, `tool_paths`, `reviewer_email`, `reviewer_role` (the artifacts-dir location lives in the `~/.config/agami/path` pointer) -- **Semantic model** (`org.yaml` + the `subject_areas//` tree under `//`; default `` is `~/agami-artifacts/`) +- **Semantic model** (`datasource.yaml` + the `subject_areas//` tree under `//`; default `` is `~/agami-artifacts/`) - **Examples library** (`//examples.yaml`) -- **Organization context** (`//ORGANIZATION.md`) — your description of what the database represents, domain terminology +- **Organization context** (`//datasource.md`) — your description of what the database represents, domain terminology - **User memory** (`/USER_MEMORY.md`) — your cross-database preferences - **Query results** (everything the assistant shows you) - **Query log** (`/local/query_log.jsonl`) — your personal record of every query you ran diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index bf288d50..c876ff5a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -59,7 +59,7 @@ rm -rf ~/.claude/plugins/cache/agami-skills # only if you also installed our e # 3. Remove your data (only if you're sure you don't want it back) # Snapshot files are intentionally immutable — chmod first so rm can delete them. chmod -R u+w ~/agami-artifacts 2>/dev/null -rm -rf ~/agami-artifacts # semantic model, examples, ORGANIZATION.md, USER_MEMORY.md, .snapshots/, .git/ +rm -rf ~/agami-artifacts # semantic model, examples, datasource.md, USER_MEMORY.md, .snapshots/, .git/ rm -rf /local # credentials, .config, charts, exports, review + examples-validation dashboards # 4. Restart Claude Code (full quit, not just close window) diff --git a/docs/usage.md b/docs/usage.md index 533f78d0..31bf3274 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -37,7 +37,7 @@ $ /agami-connect [Phase 3: validate + write] ✓ Validator passed (semantic-model schema + trust block) - ✓ Wrote the model under ~/agami-artifacts/main/ (org.yaml + subject_areas/…) + ✓ Wrote the model under ~/agami-artifacts/main/ (datasource.yaml + subject_areas/…) ✓ Snapshot pinned at .snapshots/45f0fefa2403/ ✓ git init + initial commit @@ -178,7 +178,7 @@ what the correction is actually fixing: | SQL pattern (join columns, aggregation expression, filter shape) | `examples.yaml` as a new few-shot example | | Per-column meaning, unit, sign convention, or value normalization (Male/MALE/T → "Male") | The column's `description` / `choice_field` / `caveats` in its table YAML | | Cross-DB display preference (format counts with commas, default time window) | `USER_MEMORY.md` (+ updates the seed example's SQL to demonstrate the formatting) | -| Abstract business concept tied to this DB ("gold tier means lifetime spend > $10k") | `ORGANIZATION.md` | +| Abstract business concept tied to this DB ("gold tier means lifetime spend > $10k") | `datasource.md` | | Reusable aggregation that didn't exist before ("MRR = SUM(price) WHERE plan_type='subscription'") | New `metric` in the semantic model (sign-off required — Rule 1) | The classifier surfaces its decision before writing, so you can override if it diff --git a/packages/agami-core/src/admin.py b/packages/agami-core/src/admin.py index 85143d9a..519d8816 100644 --- a/packages/agami-core/src/admin.py +++ b/packages/agami-core/src/admin.py @@ -928,7 +928,7 @@ def model_overview_html( ) content = ( '
Model
' - f'

{ui.esc(org.organization)}

' + f'

{ui.esc(org.datasource)}

' 'Read-only · edit in Claude
' f'

{ui.esc(org.description or "The deployed semantic model.")}

' f'
{stats}
' @@ -1252,18 +1252,18 @@ def model_relationships_html( def model_context_html( org: Any, memory: dict[str, str], datasource: str, datasources: list[str], **chrome: str ) -> str: - """The Domain-context page — the deployed ORGANIZATION.md rendered as (safe) markdown.""" + """The Domain-context page — the deployed datasource.md rendered as (safe) markdown.""" tree = _model_tree_html(org, datasource, datasources, active_view="context") - org_md = memory.get("organization") + org_md = memory.get("datasource") doc = ( f'
{ui.md(org_md)}
' if org_md - else '

No domain context (ORGANIZATION.md) deployed for this datasource.

' + else '

No domain context (datasource.md) deployed for this datasource.

' ) content = ( f'
{ui.esc(datasource)}' '/Domain context

Domain context

' - '

The deployed ORGANIZATION.md — the domain notes Claude reads as context. ' + '

The deployed datasource.md — the domain notes Claude reads as context. ' f"Read-only.

{doc}" ) return _model_shell(content, tree, **chrome) diff --git a/packages/agami-core/src/agami_paths.py b/packages/agami-core/src/agami_paths.py index 61ce8714..60ae1719 100644 --- a/packages/agami-core/src/agami_paths.py +++ b/packages/agami-core/src/agami_paths.py @@ -12,7 +12,7 @@ charts/, exports/ # per-query outputs model/, review/, examples-validation/ # rendered dashboards tunnels/, serve/ # ssh tunnels, the copied MCP server - / # the committable semantic model (org.yaml + subject_areas/…) + / # the committable semantic model (datasource.yaml + subject_areas/…) USER_MEMORY.md # committable cross-database preferences .gitignore # ignores local/ diff --git a/packages/agami-core/src/execute_sql.py b/packages/agami-core/src/execute_sql.py index 0b0825a7..fb0cd20a 100644 --- a/packages/agami-core/src/execute_sql.py +++ b/packages/agami-core/src/execute_sql.py @@ -896,7 +896,7 @@ def _resolve_guard_model(profile: str): pass # DB unreachable/misconfigured -> fall through to disk root = Path(os.environ.get("AGAMI_ARTIFACTS_DIR") or (Path.home() / "agami-artifacts")) / profile - if (root / "org.yaml").exists(): + if (root / "datasource.yaml").exists(): try: return L.load_organization(root) except Exception: diff --git a/packages/agami-core/src/mcp_http.py b/packages/agami-core/src/mcp_http.py index b5c2ee46..a3b21511 100644 --- a/packages/agami-core/src/mcp_http.py +++ b/packages/agami-core/src/mcp_http.py @@ -106,15 +106,15 @@ def _build_auth_provider() -> AuthProvider: def _build_org_resolver() -> SingleTenantOrgResolver: """The OSS default tenancy: single-tenant, one configured org. The id is the F14 minted uuid - resolved by `tools.resolved_org_id()` (AGAMI_ORG_ID env -> org.yaml -> "local") — the SAME + resolved by `tools.resolved_org_id()` (AGAMI_ORG_ID env -> datasource.yaml -> "local") — the SAME resolution the deploy stamp uses, so writes and reads agree. Multi-tenant is a future change at the *schema* layer plus an authz check — not a resolver swap, so the seam lives here now.""" org_id = resolved_org_id() # Load-bearing for a hand-rolled DB-only deploy: log the resolved id + where it came from, so an - # operator can see "org_id=local (default)" and realize org.yaml/AGAMI_ORG_ID isn't reaching the + # operator can see "org_id=local (default)" and realize datasource.yaml/AGAMI_ORG_ID isn't reaching the # server (see F14's documented residual risk). source = "env" if os.environ.get("AGAMI_ORG_ID", "").strip() else ( - "default" if org_id == "local" else "org.yaml" + "default" if org_id == "local" else "datasource.yaml" ) _log.info("single-tenant org_id=%s (source=%s)", org_id, source) return SingleTenantOrgResolver(Org(id=org_id)) diff --git a/packages/agami-core/src/migrations/core/001_serving.sql b/packages/agami-core/src/migrations/core/001_serving.sql index 08ef2d09..674f8307 100644 --- a/packages/agami-core/src/migrations/core/001_serving.sql +++ b/packages/agami-core/src/migrations/core/001_serving.sql @@ -99,14 +99,14 @@ CREATE TABLE prompt_example ( PRIMARY KEY (org_id, datasource, id) ); --- Domain-context docs. kind='organization' (ORGANIZATION.md) is per-datasource; kind='user' +-- Domain-context docs. kind='datasource' (datasource.md) is per-datasource; kind='user' -- (USER_MEMORY.md) is cross-datasource, stored once under datasource='' (the empty sentinel) — but -- still PER ORG, so one tenant's user memory can't reach another's. Mirrors the file layout --- (//ORGANIZATION.md vs /USER_MEMORY.md). +-- (//datasource.md vs /USER_MEMORY.md). CREATE TABLE memory ( org_id TEXT NOT NULL DEFAULT 'local', datasource TEXT NOT NULL, -- '' for the per-org user-memory row - kind TEXT NOT NULL, -- 'organization' (per-datasource) | 'user' (per-org, cross-datasource) + kind TEXT NOT NULL, -- 'datasource' (per-datasource) | 'user' (per-org, cross-datasource) content TEXT, PRIMARY KEY (org_id, datasource, kind) ); diff --git a/packages/agami-core/src/model_deploy.py b/packages/agami-core/src/model_deploy.py index 92712c7c..22890a9e 100644 --- a/packages/agami-core/src/model_deploy.py +++ b/packages/agami-core/src/model_deploy.py @@ -21,7 +21,7 @@ def _default_org() -> str: """The org to deploy under when the caller names none. A CLI has no request, so it calls the SAME resolver the server's read path uses (`tools.resolved_org_id`: AGAMI_ORG_ID -> the minted uuid in - org.yaml -> 'local') — the two MUST agree, or the model is written under one org and read under + datasource.yaml -> 'local') — the two MUST agree, or the model is written under one org and read under another and the server sees no model (F14 / ACE-056).""" from tools import resolved_org_id # lazy: keeps the deploy CLI's import surface small @@ -57,7 +57,7 @@ def _backfill_org_id(store: Store, org_id: str) -> None: def deploy_one(store: Store, datasource: str, profile_dir: Path, org_id: str | None = None) -> None: - """Load one datasource's per-datasource model (org + examples + ORGANIZATION.md + version) from + """Load one datasource's per-datasource model (org + examples + datasource.md + version) from `profile_dir` into the store. The install-global `USER_MEMORY.md` is handled once per run, separately (`_deploy_user_memory`) — it lives at the artifacts ROOT, not per profile. @@ -84,8 +84,8 @@ def deploy_one(store: Store, datasource: str, profile_dir: Path, org_id: str | N print(f"model_deploy: skipping malformed examples for area {sa.name!r}: {e}", file=sys.stderr) continue examples.extend({**ex, "area": ex.get("area") or sa.name} for ex in area_examples if isinstance(ex, dict)) - org_md = profile_dir / "ORGANIZATION.md" - org_text = org_md.read_text() if org_md.exists() else None + datasource_md = profile_dir / "datasource.md" + datasource_text = datasource_md.read_text() if datasource_md.exists() else None version = newest_version(profile_dir) or "deployed" # --- then write (version last, so its presence marks a completed deploy) --- @@ -94,7 +94,7 @@ def deploy_one(store: Store, datasource: str, profile_dir: Path, org_id: str | N # write_examples is clear-then-insert, so an empty list replaces the datasource's examples with none. model_store.write_examples(store, datasource, examples, org_id=org_id) model_store.write_memory( - store, datasource, organization=org_text, org_id=org_id + store, datasource, datasource_doc=datasource_text, org_id=org_id ) # per-datasource model_store.write_model_version(store, datasource, version, org_id=org_id) store.commit() @@ -130,20 +130,20 @@ def _deploy_org_record(store: Store, artifacts_dir: Path, org_id: str | None = N # (datasource='' sentinel, like USER_MEMORY.md) so the served two-level context can read it. narrative = OR.narrative_path(artifacts_dir) if narrative.exists(): - model_store.write_memory(store, "", organization=narrative.read_text(), org_id=org_id) + model_store.write_memory(store, "", datasource_doc=narrative.read_text(), org_id=org_id) def deploy_models(store: Store, artifacts_dir: Path, org_id: str | None = None) -> list[str]: - """Load every datasource model under `artifacts_dir` (a *directory* with an `org.yaml`) into the store. + """Load every datasource model under `artifacts_dir` (a *directory* with an `datasource.yaml`) into the store. Returns the datasources loaded. The `local/` dir (gitignored secrets/state) and any non-directory or - org.yaml-less entry are skipped — `local/` explicitly, so a stray `local/org.yaml` can't deploy from the + datasource.yaml-less entry are skipped — `local/` explicitly, so a stray `local/datasource.yaml` can't deploy from the secrets dir.""" org_id = org_id if org_id is not None else _default_org() loaded: list[str] = [] for prof in sorted( p for p in artifacts_dir.iterdir() - if p.is_dir() and p.name != agami_paths.LOCAL_SUBDIR and (p / "org.yaml").exists() + if p.is_dir() and p.name != agami_paths.LOCAL_SUBDIR and (p / "datasource.yaml").exists() ): deploy_one(store, prof.name, prof, org_id=org_id) loaded.append(prof.name) @@ -174,9 +174,9 @@ def main(argv: list[str] | None = None) -> int: loaded: list[str] = [] for ds in args: prof = artifacts_dir / ds - if not (prof / "org.yaml").exists(): + if not (prof / "datasource.yaml").exists(): print( - f"model_deploy: no model for datasource {ds!r} at {prof}/org.yaml", + f"model_deploy: no model for datasource {ds!r} at {prof}/datasource.yaml", file=sys.stderr, ) return 1 @@ -187,7 +187,7 @@ def main(argv: list[str] | None = None) -> int: if not loaded: print( f"model_deploy: no model found under {artifacts_dir} " - "(a datasource is a subdir with an org.yaml).", + "(a datasource is a subdir with an datasource.yaml).", file=sys.stderr, ) return 1 diff --git a/packages/agami-core/src/model_store.py b/packages/agami-core/src/model_store.py index 7af40d07..a6db2bca 100644 --- a/packages/agami-core/src/model_store.py +++ b/packages/agami-core/src/model_store.py @@ -171,14 +171,14 @@ def model_table_counts(store: Store, org_id: str = DEFAULT_ORG) -> dict[str, int # --------------------------------------------------------------------------- -# Memory (ORGANIZATION.md / USER_MEMORY.md) + model_version — served from the DB too, so a DB-only +# Memory (datasource.md / USER_MEMORY.md) + model_version — served from the DB too, so a DB-only # deploy reads NO files at runtime (get_datasource_schema's domain context + the receipt's version # pin come from these tables, not disk). # --------------------------------------------------------------------------- -# ORGANIZATION.md is per-datasource; USER_MEMORY.md is cross-datasource (mirroring the file layout: -# //ORGANIZATION.md vs /USER_MEMORY.md), so it is stored once +# datasource.md is per-datasource; USER_MEMORY.md is cross-datasource (mirroring the file layout: +# //datasource.md vs /USER_MEMORY.md), so it is stored once # under this sentinel datasource rather than duplicated per datasource. It is still keyed by org — one # row PER ORG, not one per install, or one tenant's user memory would be served to another's. _GLOBAL_DATASOURCE = "" @@ -188,21 +188,21 @@ def write_memory( store: Store, datasource: str, *, - organization: str | None = None, + datasource_doc: str | None = None, user: str | None = None, org_id: str = DEFAULT_ORG, ) -> None: - """Seed the domain-context docs. `organization` is per-datasource; `user` is cross-datasource but + """Seed the domain-context docs. `datasource_doc` is per-datasource; `user` is cross-datasource but still per-org (the empty-datasource sentinel row). Pass either/both; each replaces its row.""" - if organization is not None: + if datasource_doc is not None: store.execute( - "DELETE FROM memory WHERE org_id = ? AND datasource = ? AND kind = 'organization'", + "DELETE FROM memory WHERE org_id = ? AND datasource = ? AND kind = 'datasource'", (org_id, datasource), ) store.execute( "INSERT INTO memory (org_id, datasource, kind, content) " - "VALUES (?, ?, 'organization', ?)", - (org_id, datasource, organization), + "VALUES (?, ?, 'datasource', ?)", + (org_id, datasource, datasource_doc), ) if user is not None: store.execute( @@ -217,10 +217,10 @@ def write_memory( def load_memory(store: Store, datasource: str, org_id: str = DEFAULT_ORG) -> dict[str, str]: - """{'organization': , 'user': } — + """{'datasource': , 'user': } — missing keys absent.""" out: dict[str, str] = {} - for kind, ds in (("organization", datasource), ("user", _GLOBAL_DATASOURCE)): + for kind, ds in (("datasource", datasource), ("user", _GLOBAL_DATASOURCE)): rows = store.query( "SELECT content FROM memory WHERE org_id = ? AND datasource = ? AND kind = ?", (org_id, ds, kind), diff --git a/packages/agami-core/src/semantic_model/__init__.py b/packages/agami-core/src/semantic_model/__init__.py index 78732c5d..55a107fd 100644 --- a/packages/agami-core/src/semantic_model/__init__.py +++ b/packages/agami-core/src/semantic_model/__init__.py @@ -9,7 +9,7 @@ Layout on disk (the canonical profile format, rooted at `//`): - org.yaml # org desc + storage_connections + subject_areas + datasource.yaml # org desc + storage_connections + subject_areas datasources//storage.yaml # physical: storage_type, storage_config subject_areas// subject_area.yaml # desc, default_time_window, tables (TableRefs) diff --git a/packages/agami-core/src/semantic_model/build.py b/packages/agami-core/src/semantic_model/build.py index 173a9edb..cf499ec2 100644 --- a/packages/agami-core/src/semantic_model/build.py +++ b/packages/agami-core/src/semantic_model/build.py @@ -481,7 +481,7 @@ class WriteReport: def ensure_org_id(out: Path, existing: Optional[str] = None, *, dry_run: bool = False) -> str: """Idempotent, deployment-scoped resolution of the org identity (F14 / ACE-056; relocated by F15 / ACE-067). Returns, in order: ``existing`` (an id the caller already resolved); the org_id - already persisted at ``out/org.yaml`` (preserved across re-introspect — the immutability guarantee); + already persisted at ``out/datasource.yaml`` (preserved across re-introspect — the immutability guarantee); else the DEPLOYMENT record's id (``organization.yaml`` at ``out.parent``), minting the record on first use. @@ -511,7 +511,7 @@ def write_tree( examples_by_area: Optional[dict[str, list[dict]]] = None, dry_run: bool = False, ) -> WriteReport: - """Write the canonical on-disk tree (org.yaml + datasources + subject_areas + + """Write the canonical on-disk tree (datasource.yaml + datasources + subject_areas + optional prompt_examples). Returns the list of files (would-be on dry_run).""" examples_by_area = examples_by_area or {} rep = WriteReport(out_dir=str(out), dry_run=dry_run) @@ -524,14 +524,14 @@ def write(rel: str, content: str) -> None: p.parent.mkdir(parents=True, exist_ok=True) p.write_text(content, encoding="utf-8") - # F14/F15: the deployment identity is the first key. `ensure_org_id` reads the *previous* org.yaml + # F14/F15: the deployment identity is the first key. `ensure_org_id` reads the *previous* datasource.yaml # (about to be overwritten) so a re-introspect/curate/snapshot preserves the minted id rather than # re-minting — the immutability guarantee — and otherwise takes it from the deployment record # (minting the record on first use). Resolved once, honoring dry_run so a preview persists nothing. org_id = ensure_org_id(out, org.org_id, dry_run=dry_run) - write("org.yaml", _dump({ + write("datasource.yaml", _dump({ "org_id": org_id, - "organization": org.organization, + "datasource": org.datasource, "version": org.version, "description": org.description, "fiscal_year_start_month": org.fiscal_year_start_month, diff --git a/packages/agami-core/src/semantic_model/cli.py b/packages/agami-core/src/semantic_model/cli.py index 8f54d0d8..3042e2e7 100644 --- a/packages/agami-core/src/semantic_model/cli.py +++ b/packages/agami-core/src/semantic_model/cli.py @@ -87,7 +87,7 @@ def cmd_snapshot(args) -> int: def cmd_ensure_org_id(args) -> int: - """Mint + persist the deployment org_id into /org.yaml if absent (F14 / ACE-056; relocated + """Mint + persist the deployment org_id into /datasource.yaml if absent (F14 / ACE-056; relocated by F15 / ACE-067), then print it. Idempotent — an already-minted id is returned unchanged. The introspect/curate paths mint inline via write_tree; this command is for paths that write a model WITHOUT them — e.g. agami-connect's sample copy (6A) drops a prebuilt model that carries no id. @@ -97,7 +97,7 @@ def cmd_ensure_org_id(args) -> int: from . import build root = Path(args.root) - org_path = root / "org.yaml" + org_path = root / "datasource.yaml" doc = L._read_yaml(org_path) or {} existing = doc.get("org_id") oid = build.ensure_org_id(root, existing or None) @@ -134,7 +134,7 @@ def cmd_areas(args) -> int: def cmd_org_draft(args) -> int: - # A human-narrative STARTER for ORGANIZATION.md (skip path) — a prompt only, no model + # A human-narrative STARTER for datasource.md (skip path) — a prompt only, no model # facts. Those are derived at read time (see `org-context`), so they never get baked into # the editable prose file where a human could clobber them. from . import org_draft @@ -145,14 +145,14 @@ def cmd_org_draft(args) -> int: def cmd_org_context(args) -> int: # The full domain context for the LLM, two-level (F15 / ACE-069): the shared COMPANY block from the - # deployment record (/organization.yaml + the root ORGANIZATION.md), then this + # deployment record (/organization.yaml + the root datasource.md), then this # datasource's source-specific narrative + model-derived summary. With no record it degrades to the # pre-F15 per-profile assembly (byte-identical). This is what the query path injects as context. from . import org_draft from . import org_record as OR root = Path(args.root) org = L.load_organization(root, include_rejected=False) - src_md = root / "ORGANIZATION.md" + src_md = root / "datasource.md" source_narrative = src_md.read_text(encoding="utf-8") if src_md.exists() else "" art = root.parent # the artifacts dir holds the deployment-level record + company narrative record = OR.load_org_record(art) @@ -326,7 +326,7 @@ def cmd_sensitive(args) -> int: def cmd_set_terminology(args) -> int: - """Write the org-level domain glossary (term -> definition) onto org.yaml's + """Write the org-level domain glossary (term -> definition) onto datasource.yaml's `key_terminology` — the decoded-abbreviation legend enrichment produces. Merges by default (layers over a human's edits); --replace overwrites. Validated + committed.""" from . import curate @@ -1100,7 +1100,7 @@ def build_parser() -> argparse.ArgumentParser: sp.add_argument("root") sp.set_defaults(func=cmd_snapshot) - sp = sub.add_parser("ensure-org-id", help="mint + persist the deployment org_id into org.yaml if absent (e.g. after copying a sample model)") + sp = sub.add_parser("ensure-org-id", help="mint + persist the deployment org_id into datasource.yaml if absent (e.g. after copying a sample model)") sp.add_argument("root") sp.set_defaults(func=cmd_ensure_org_id) @@ -1125,7 +1125,7 @@ def build_parser() -> argparse.ArgumentParser: sp.add_argument("root") sp.set_defaults(func=cmd_areas) - sp = sub.add_parser("org-draft", help="print a human-narrative STARTER for ORGANIZATION.md (prompt only, no facts)") + sp = sub.add_parser("org-draft", help="print a human-narrative STARTER for datasource.md (prompt only, no facts)") sp.add_argument("root") sp.set_defaults(func=cmd_org_draft) @@ -1197,7 +1197,7 @@ def build_parser() -> argparse.ArgumentParser: sp.add_argument("root") sp.set_defaults(func=cmd_sensitive) - sp = sub.add_parser("set-terminology", help="write the org domain glossary (term→definition) onto org.yaml key_terminology") + sp = sub.add_parser("set-terminology", help="write the org domain glossary (term→definition) onto datasource.yaml key_terminology") sp.add_argument("root") sp.add_argument("--file", required=True, help="JSON object {term: definition, ...} (or {key_terminology: {...}})") sp.add_argument("--replace", action="store_true", help="replace the glossary instead of merging over it") @@ -1367,7 +1367,7 @@ def main(argv=None) -> int: # callers fold the "does a model exist?" check into their first real command # (e.g. `sm areas`) instead of a separate filesystem probe. Exit 3 = "no model # here — run agami-connect"; distinct from validation errors (1) and usage (2). - if "org.yaml" in str(e): + if "datasource.yaml" in str(e): _print_json({"error": "no_model", "detail": str(e)}) return 3 raise diff --git a/packages/agami-core/src/semantic_model/curate.py b/packages/agami-core/src/semantic_model/curate.py index 149fe525..de3a5a49 100644 --- a/packages/agami-core/src/semantic_model/curate.py +++ b/packages/agami-core/src/semantic_model/curate.py @@ -375,7 +375,7 @@ def model_tree(org: Organization) -> dict[str, Any]: for c in t.columns], }) areas.append({"area": sa.name, "description": sa.description, "tables": tables}) - return {"organization": org.organization, "subject_areas": areas} + return {"datasource": org.datasource, "subject_areas": areas} # --------------------------------------------------------------------------- @@ -414,18 +414,18 @@ def _snapshot(backups: "dict[Path, Optional[str]]", path: Path) -> None: def set_key_terminology(root: str | Path, terms: dict, *, merge: bool = True) -> "ApplyResult": - """Write the org-level domain glossary (term -> definition) onto org.yaml's + """Write the org-level domain glossary (term -> definition) onto datasource.yaml's `key_terminology`. Validated + git-committed like every other write; the prior - org.yaml is restored on validation failure (no git dependency for the revert). + datasource.yaml is restored on validation failure (no git dependency for the revert). `merge=True` (default) layers `terms` over the existing glossary — so an enrichment pass adds without clobbering a human's edits; `merge=False` replaces. Empty terms/definitions are dropped.""" root = Path(root) res = ApplyResult() - orgp = root / "org.yaml" + orgp = root / "datasource.yaml" if not orgp.exists(): - res.errors.append(f"no org.yaml at {orgp}") + res.errors.append(f"no datasource.yaml at {orgp}") return res prior = orgp.read_text(encoding="utf-8") odoc = _load(orgp) or {} @@ -631,8 +631,8 @@ def _apply_one(root: Path, op: dict, signer, role, _dump(path, {"relationships": rels}) return path # Cross-area (cross-schema / cross-datasource) join — it lives at the org level, not - # in an area's relationships.yaml. Fall back to org.yaml's cross_subject_area_relationships. - orgp = root / "org.yaml" + # in an area's relationships.yaml. Fall back to datasource.yaml's cross_subject_area_relationships. + orgp = root / "datasource.yaml" _snapshot(backups, orgp) odoc = _load(orgp) or {} crels = odoc.get("cross_subject_area_relationships", []) diff --git a/packages/agami-core/src/semantic_model/introspect.py b/packages/agami-core/src/semantic_model/introspect.py index edb870be..4127ce0c 100644 --- a/packages/agami-core/src/semantic_model/introspect.py +++ b/packages/agami-core/src/semantic_model/introspect.py @@ -278,7 +278,7 @@ def introspect( # edits to lose); the relationship pass below rebuilds across the union, prior edges kept. prev_rels: list[Relationship] = [] skip_keys: set = set() - if append and not dry_run and (out / "org.yaml").exists(): + if append and not dry_run and (out / "datasource.yaml").exists(): from .loader import load_organization prev = load_organization(out, include_rejected=True) batch_names = {t.name for t in built} @@ -324,7 +324,7 @@ def introspect( storage_config={"profile": profile, "credentials_ref": "/local/credentials"}, ) org = Organization( - organization=profile, + datasource=profile, version=1, storage_connections=[storage], subject_areas=areas, diff --git a/packages/agami-core/src/semantic_model/loader.py b/packages/agami-core/src/semantic_model/loader.py index 863c7467..86b5ab5f 100644 --- a/packages/agami-core/src/semantic_model/loader.py +++ b/packages/agami-core/src/semantic_model/loader.py @@ -4,7 +4,7 @@ ``//``: / - org.yaml # org desc + storage_connections + subject_areas refs + datasource.yaml # org desc + storage_connections + subject_areas refs datasources//storage.yaml # physical: storage_type, storage_config subject_areas// subject_area.yaml # desc, default_time_window, tables (TableRefs) @@ -84,9 +84,9 @@ def load_organization(root: str | Path, *, include_rejected: bool = False) -> Or tools (agami-model), which must show excluded entries to toggle them. """ root = Path(root) - org_path = root / "org.yaml" + org_path = root / "datasource.yaml" if not org_path.exists(): - raise FileNotFoundError(f"no org.yaml at {org_path}") + raise FileNotFoundError(f"no datasource.yaml at {org_path}") org_doc: dict[str, Any] = _read_yaml(org_path) or {} # storage connections — inline list OR refs into datasources//storage.yaml @@ -115,7 +115,7 @@ def load_organization(root: str | Path, *, include_rejected: bool = False) -> Or org = Organization( org_id=org_doc.get("org_id"), # F14: minted uuid4 (None for pre-F14 files) - organization=org_doc.get("organization", root.name), + datasource=org_doc.get("datasource", root.name), version=org_doc.get("version", 1), description=org_doc.get("description", ""), fiscal_year_start_month=org_doc.get("fiscal_year_start_month", 1), @@ -130,14 +130,14 @@ def load_organization(root: str | Path, *, include_rejected: bool = False) -> Or def load_org_id(root: str | Path) -> str | None: - """Return the minted org_id recorded in ``/org.yaml``, or ``None`` if the file is absent + """Return the minted org_id recorded in ``/datasource.yaml``, or ``None`` if the file is absent or predates F14 (no ``org_id`` key). This is the serve-time identity read (F14 / ACE-056). Deliberately read-only and lenient — unlike ``load_organization`` it never raises on a missing file — so the single-tenant resolver can fall through to its ``"local"`` default. Reads only the top-level key; it does NOT build the full pydantic model (identity resolution runs per process, not per query, so it stays cheap).""" - org_path = Path(root) / "org.yaml" + org_path = Path(root) / "datasource.yaml" if not org_path.exists(): return None doc = _read_yaml(org_path) or {} @@ -146,11 +146,11 @@ def load_org_id(root: str | Path) -> str | None: def deployment_org_id(artifacts_dir: str | Path) -> str | None: - """The deployment-wide org identity: the first ``org_id`` found across ``/*/org.yaml``, + """The deployment-wide org identity: the first ``org_id`` found across ``/*/datasource.yaml``, or ``None`` if no profile carries one (F14 / ACE-056, the *deployment-scoped* rule). A deployment is ONE tenant even with several datasource profiles, so the minted id is shared across - every profile's ``org.yaml``. Resolving by scanning the artifacts dir (rather than one 'active' + every profile's ``datasource.yaml``. Resolving by scanning the artifacts dir (rather than one 'active' profile) means the deploy stamp and the serve resolver agree even when ``AGAMI_PROFILE`` is unset and the real model lives under a named profile (e.g. ``northpeak_salesforce``, not ``default``). Read-only; never raises. Profiles are expected to agree; the first (sorted) is returned deterministically.""" @@ -279,7 +279,7 @@ def _load_subject_area(sa_dir: Path, include_rejected: bool = False) -> SubjectA def _load_cross_rels(root: Path, org_doc: dict) -> list[CrossSubjectAreaRelationship]: out: list[CrossSubjectAreaRelationship] = [] - # inline on org.yaml + # inline on datasource.yaml for r in org_doc.get("cross_subject_area_relationships", []) or []: out.append(CrossSubjectAreaRelationship(**r)) # or a sidecar file diff --git a/packages/agami-core/src/semantic_model/models.py b/packages/agami-core/src/semantic_model/models.py index cca23cb2..04d5d67f 100644 --- a/packages/agami-core/src/semantic_model/models.py +++ b/packages/agami-core/src/semantic_model/models.py @@ -633,11 +633,11 @@ def defined_table(self, name: str) -> Optional[Table]: class Organization(_Base): # Locally-minted, globally-unique deployment identity (F14 / ACE-056): a uuid4 hex minted ONCE at - # agami-connect and persisted in org.yaml, then immutable. Optional/None so pre-F14 org.yaml files + # agami-connect and persisted in datasource.yaml, then immutable. Optional/None so pre-F14 datasource.yaml files # (which have no org_id key) still load under the model's `extra="forbid"` policy. Never transmitted; # it only travels the day the operator exports their own DB — see F14's no-egress invariant. org_id: Optional[str] = None - organization: str + datasource: str # this datasource's display name (used as the model title in the explorer/context) version: int = 1 description: str = "" fiscal_year_start_month: int = 1 @@ -651,8 +651,8 @@ class Organization(_Base): cross_subject_area_metrics: list[Metric] = Field(default_factory=list) # domain glossary: term -> one-line definition (e.g. "MRR": "monthly recurring revenue"). # Enrichment fills this from decoded abbreviations + choice-field legends; org_draft - # renders it into ORGANIZATION.md's "Key terminology", and it feeds NL→SQL as context. - # The structured home means it survives an ORGANIZATION.md regeneration (a prose section + # renders it into datasource.md's "Key terminology", and it feeds NL→SQL as context. + # The structured home means it survives an datasource.md regeneration (a prose section # the LLM has to remember to write does not). key_terminology: dict[str, str] = Field(default_factory=dict) @@ -697,8 +697,8 @@ class OrgRecord(_Base): """The deployment-level company record (F15). Written ONCE at ``/organization.yaml`` and shared across every datasource under the deployment, so company-wide facts (name, description, fiscal year, display conventions, glossary) live in one place instead of drifting across each - profile's ``org.yaml``. ``org_id`` is F14's deployment identity, relocated here from the per-profile - ``org.yaml`` (minted once, immutable, deployment-scoped — the value is preserved, never re-minted). + profile's ``datasource.yaml``. ``org_id`` is F14's deployment identity, relocated here from the per-profile + ``datasource.yaml`` (minted once, immutable, deployment-scoped — the value is preserved, never re-minted). Every non-id field is optional so a bare record (id only) validates; company content is authored later by onboarding/curation.""" diff --git a/packages/agami-core/src/semantic_model/org_draft.py b/packages/agami-core/src/semantic_model/org_draft.py index 5f7f2472..2662fa7e 100644 --- a/packages/agami-core/src/semantic_model/org_draft.py +++ b/packages/agami-core/src/semantic_model/org_draft.py @@ -2,13 +2,13 @@ The architecture deliberately keeps the human's words and the auto-derived facts apart: -* **ORGANIZATION.md** is the human's narrative ONLY (what the company/product is, who the +* **datasource.md** is the human's narrative ONLY (what the company/product is, who the users are). agami never writes facts into it, so there is nothing for a human to accidentally overwrite or delete. `starter_organization_md()` is the blank-path prompt. * **The factual context** — shape, subject areas, conventions, and the decoded glossary — is `derived_context()`, computed FRESH from the structured model at read time. The glossary lives in the structured `key_terminology` field, not inline prose, so it always reaches the - LLM regardless of what's in (or missing from) ORGANIZATION.md. + LLM regardless of what's in (or missing from) datasource.md. `compose_context(human_md, org)` assembles the two for a reader (MCP / query skill / explorer): the human's narrative, then the derived summary under its own heading. Never a re-listing of @@ -50,7 +50,7 @@ def _strip_comments(text: str) -> str: def derived_context(org: "Organization", *, with_curated_glossary: bool = True) -> str: """The model-DERIVED factual context: shape + subject areas + conventions + glossary. - Computed fresh from the structured model every time and NOT persisted into ORGANIZATION.md + Computed fresh from the structured model every time and NOT persisted into datasource.md — so there are no fragile markers for a human to clobber, and the glossary always reaches the LLM (it no longer depends on a file having been re-rendered). The glossary comes from the structured `key_terminology` field + `choice_field` enum legends. Never a re-listing @@ -65,7 +65,7 @@ def derived_context(org: "Organization", *, with_curated_glossary: bool = True) n_entities = sum(len(sa.entities) for sa in areas) + len(org.cross_subject_area_entities) lines: list[str] = [ - f"**{org.organization}** — {_plural(n_tables, 'table')} across {_plural(len(areas), 'subject area')}.", + f"**{org.datasource}** — {_plural(n_tables, 'table')} across {_plural(len(areas), 'subject area')}.", "", ] if areas: @@ -119,7 +119,7 @@ def compose_context(human_md: str, org: "Organization") -> str: def _company_block(record: "OrgRecord", narrative: str) -> str: - """The shared COMPANY context (F15): name/description + the root ORGANIZATION.md narrative + + """The shared COMPANY context (F15): name/description + the root datasource.md narrative + company-wide display conventions + the company glossary. Rendered ONCE above every datasource.""" lines: list[str] = [f"# {record.name or 'Company'} — company context", ""] if record.description: @@ -156,7 +156,7 @@ def _company_block(record: "OrgRecord", narrative: str) -> str: def _source_block(org: "Organization", narrative: str) -> str: """One datasource's context: its optional source-specific narrative + the model-derived summary, under a heading naming the datasource (so a federated answer keeps the vocabularies apart).""" - seg = [f"## {org.organization} — datasource context"] + seg = [f"## {org.datasource} — datasource context"] src = _strip_comments(narrative) if src: seg.append(src) @@ -174,13 +174,13 @@ def compose_org_context( source_narratives: "list[str] | None" = None, ) -> str: """Two-level org context (F15 / ACE-069). Renders the shared COMPANY block ONCE from the ``OrgRecord`` - (name/description + the root ``ORGANIZATION.md`` narrative + display conventions + company glossary), + (name/description + the root ``datasource.md`` narrative + display conventions + company glossary), then each datasource's model-derived ontology under its own heading. Single-source passes one ontology; a FEDERATED question passes several — the company block still renders exactly once, and each source keeps its own vocabulary (so "Account" resolves per source). ``source_narratives`` (optional, aligned by index to ``ontologies``) carries each datasource's - source-specific ``ORGANIZATION.md`` prose; company-wide prose now lives in ``company_narrative`` (the + source-specific ``datasource.md`` prose; company-wide prose now lives in ``company_narrative`` (the root file), not the per-profile ones. Graceful degradation: when ``org_record`` is ``None`` this returns exactly today's per-profile @@ -209,9 +209,9 @@ def starter_organization_md(org: "Organization") -> str: areas = list(org.subject_areas) n_tables = sum(len(sa.tables_defined) for sa in areas) names = [sa.name for sa in areas] - lead = f"**{org.organization}** holds {_plural(n_tables, 'table')} across {_plural(len(areas), 'subject area')}" + lead = f"**{org.datasource}** holds {_plural(n_tables, 'table')} across {_plural(len(areas), 'subject area')}" # Name the areas when they add signal (skip a lone area that just echoes the org name). - if names and not (len(names) == 1 and names[0].lower() == org.organization.lower()): + if names and not (len(names) == 1 and names[0].lower() == org.datasource.lower()): shown = ", ".join(names[:6]) + (f", and {len(names) - 6} more" if len(names) > 6 else "") lead += f" covering {shown}." else: diff --git a/packages/agami-core/src/semantic_model/org_record.py b/packages/agami-core/src/semantic_model/org_record.py index 922c8463..8ff1c8b7 100644 --- a/packages/agami-core/src/semantic_model/org_record.py +++ b/packages/agami-core/src/semantic_model/org_record.py @@ -1,9 +1,9 @@ """The deployment-level organization record (F15 / ACE-067). One ``OrgRecord`` lives at ``/organization.yaml`` — ABOVE the per-profile -``//org.yaml`` models — and holds the company-wide facts (name, description, +``//datasource.yaml`` models — and holds the company-wide facts (name, description, fiscal year, display conventions, glossary) that would otherwise be duplicated into every profile's -``org.yaml`` and drift. The company narrative lives beside it at ``/ORGANIZATION.md``. +``datasource.yaml`` and drift. The company narrative lives beside it at ``/datasource.md``. This module owns: @@ -29,7 +29,7 @@ # The record and the company narrative both sit at the artifacts-dir ROOT (one deployment = one company), # NOT under a profile dir — that is the whole point: written once, shared by every datasource. RECORD_FILENAME = "organization.yaml" -NARRATIVE_FILENAME = "ORGANIZATION.md" +NARRATIVE_FILENAME = "datasource.md" def record_path(artifacts_dir: str | Path) -> Path: @@ -54,10 +54,10 @@ def load_org_record(artifacts_dir: str | Path) -> Optional[OrgRecord]: def ensure_org_record(artifacts_dir: str | Path) -> OrgRecord: """Read the deployment's ``OrgRecord``, or mint a fresh one and persist it. The ``org_id`` mint - chokepoint (relocated here from the per-profile ``org.yaml`` — F14's ``ensure_org_id``): + chokepoint (relocated here from the per-profile ``datasource.yaml`` — F14's ``ensure_org_id``): 1. an existing ``organization.yaml`` is returned unchanged (mint-once / immutable); - 2. else, if a profile ``org.yaml`` already carries an id (a post-F14 deployment), that id is + 2. else, if a profile ``datasource.yaml`` already carries an id (a post-F14 deployment), that id is LIFTED up into a new record — never re-minted (preserves F14's immutable value); 3. else a fresh ``uuid4().hex`` is minted into a new record. @@ -74,7 +74,7 @@ def ensure_org_record(artifacts_dir: str | Path) -> OrgRecord: def write_org_record(artifacts_dir: str | Path, record: OrgRecord) -> Path: """Persist ``record`` to ``/organization.yaml`` (creating the dir if needed) and - return the path. Written with the same default permissions as the sibling ``org.yaml`` — the record + return the path. Written with the same default permissions as the sibling ``datasource.yaml`` — the record holds company context, not secrets, and mode-600 model files are unreadable by the deploy container user (a known crash-loop), so this deliberately does NOT ``chmod 600``.""" path = record_path(artifacts_dir) @@ -107,12 +107,12 @@ def set_org_fields( def refresh_datasources(artifacts_dir: str | Path) -> Optional[OrgRecord]: """Rebuild the record's ``datasources`` list from the profile directories actually present on disk - (each immediate subdir holding an ``org.yaml``), so the list is auto-maintained and can never drift. + (each immediate subdir holding an ``datasource.yaml``), so the list is auto-maintained and can never drift. Returns ``None`` (and writes nothing) when there is neither a record nor any profile yet; otherwise mints the record if needed, updates the list, persists, and returns it.""" art = Path(artifacts_dir) names = ( - sorted(p.name for p in art.iterdir() if p.is_dir() and (p / "org.yaml").exists()) + sorted(p.name for p in art.iterdir() if p.is_dir() and (p / "datasource.yaml").exists()) if art.is_dir() else [] ) diff --git a/packages/agami-core/src/semantic_model/snapshot.py b/packages/agami-core/src/semantic_model/snapshot.py index 48be6db1..7c15c7dc 100644 --- a/packages/agami-core/src/semantic_model/snapshot.py +++ b/packages/agami-core/src/semantic_model/snapshot.py @@ -109,7 +109,7 @@ def write_snapshot(root: str | os.PathLike) -> str | None: `model_version`), or None if there's no model / on any error (best-effort).""" try: root = Path(root) - if not (root / "org.yaml").exists(): + if not (root / "datasource.yaml").exists(): return None # not a model root — nothing to snapshot # One pass: the hash (to name the dir) and the manifest shas (written only on a new dir) # come from a single read of each file, not two. diff --git a/packages/agami-core/src/tools.py b/packages/agami-core/src/tools.py index d650f020..75e772cd 100644 --- a/packages/agami-core/src/tools.py +++ b/packages/agami-core/src/tools.py @@ -229,9 +229,9 @@ def _load_org(profile: str): from semantic_model import loader as L # may raise ImportError (pydantic) root = resolve_artifacts_dir() / profile - if not (root / "org.yaml").exists(): + if not (root / "datasource.yaml").exists(): raise FileNotFoundError( - f"No semantic model at {root}/org.yaml. Run the agami-connect skill to " + f"No semantic model at {root}/datasource.yaml. Run the agami-connect skill to " f"introspect this database." ) return L.load_organization(root) @@ -289,7 +289,7 @@ def resolved_org_id() -> str: The SAME function backs both the deploy-time stamp (``model_deploy._default_org``) and the serve-time resolver, so a deployment writes and reads its rows under one identical id. - F15 relocates the id's home from each profile's ``org.yaml`` up into the one root record, so the + F15 relocates the id's home from each profile's ``datasource.yaml`` up into the one root record, so the org owns its own identity; the per-profile scan is kept only as the legacy fallback for a deployment that has a per-profile id but no record yet (the id is lifted into a record on the next onboard). The artifacts-dir scope (not one 'active' profile) is preserved: a deploy with ``AGAMI_PROFILE`` @@ -306,7 +306,7 @@ def resolved_org_id() -> str: art = resolve_artifacts_dir() record = OR.load_org_record(art) # F15: the record is the home of the id - # Legacy fallback: a pre-record deployment still keeps its id in each profile's org.yaml. + # Legacy fallback: a pre-record deployment still keeps its id in each profile's datasource.yaml. oid = record.org_id if record is not None else L.deployment_org_id(art) except Exception: oid = None # missing/legacy record or absent model deps -> single-tenant default @@ -384,7 +384,7 @@ def get_cached_org(profile: str): def _context_sources(profile: str, org_id: str) -> "tuple[str, str | None, Any, str]": """Every piece of domain-context text the served schema needs, read in ONE place: the per-datasource - ORGANIZATION.md, USER_MEMORY.md, the deployment ``OrgRecord``, and the company narrative. Under the DB + datasource.md, USER_MEMORY.md, the deployment ``OrgRecord``, and the company narrative. Under the DB backend all of it is read on a SINGLE connection — this is a hot tool path, so open ``Store`` once, not per-source; with no DB configured it falls back to file reads (a DB deploy reads no files at runtime). Returns ``(org_md, user_md, record | None, company_md)``; missing pieces come back empty/``None`` so the @@ -396,18 +396,18 @@ def _context_sources(profile: str, org_id: str) -> "tuple[str, str | None, Any, from model_store import load_memory, load_organization_record try: - mem = load_memory(store, profile, org_id=org_id) # per-datasource ORGANIZATION.md + USER_MEMORY.md + mem = load_memory(store, profile, org_id=org_id) # per-datasource datasource.md + USER_MEMORY.md record = load_organization_record(store, org_id) # the deployment company record company = load_memory(store, "", org_id=org_id) # company narrative rides the datasource='' row finally: store.close() - return mem.get("organization") or "", mem.get("user"), record, (company.get("organization") or "") + return mem.get("datasource") or "", mem.get("user"), record, (company.get("datasource") or "") from semantic_model import org_record as OR art = resolve_artifacts_dir() return ( - _read_text(art / profile / "ORGANIZATION.md") or "", + _read_text(art / profile / "datasource.md") or "", _read_text(art / "USER_MEMORY.md"), OR.load_org_record(art), _read_text(OR.narrative_path(art)) or "", @@ -466,7 +466,7 @@ def tool_list_datasources(_args: dict[str, Any]) -> str: { "datasources": [], "note": "No models deployed to this server yet. Load one with the deploy's model " - "loader (model_deploy scans /*/org.yaml).", + "loader (model_deploy scans /*/datasource.yaml).", }, indent=2, ) @@ -489,7 +489,7 @@ def tool_list_datasources(_args: dict[str, Any]) -> str: "datasource": profile, "database_type": _db_type_for(profile, creds), "table_count": table_count, - "model_present": (pdir / "org.yaml").exists(), + "model_present": (pdir / "datasource.yaml").exists(), "is_active": profile == active, } ) @@ -522,7 +522,7 @@ def _read_text(path: Path) -> str | None: def _distill_for_llm(text: str | None) -> str: - """Strip the human-only scaffolding from a context doc (ORGANIZATION.md / USER_MEMORY.md) + """Strip the human-only scaffolding from a context doc (datasource.md / USER_MEMORY.md) before it goes into the model's prompt. These files serve two readers: a human editing them (who wants the `` prompts) and the LLM reading them as query context (for whom those prompts are noise — or worse, a "this was auto-generated" aside it @@ -769,7 +769,7 @@ def tool_get_datasource_schema(args: dict[str, Any]) -> str: `get_table_context` for the named tables (an explicit scope is respected — no downgrade). `query=""` lexically ranks metrics so the client never ingests the whole catalog; `metric_index` (name->description for EVERY metric) + `large_tables` are always present. - Plus ORGANIZATION.md / USER_MEMORY.md domain context. + Plus datasource.md / USER_MEMORY.md domain context. """ profile = resolve_profile(args.get("datasource")) try: @@ -846,9 +846,9 @@ def tool_get_datasource_schema(args: dict[str, Any]) -> str: ) parts = [json.dumps(result, indent=2, default=str)] - # Domain context = the human's ORGANIZATION.md narrative + the model-DERIVED summary + # Domain context = the human's datasource.md narrative + the model-DERIVED summary # (subject areas, conventions, decoded glossary) assembled fresh from the structured model. - # Source (ORGANIZATION.md / USER_MEMORY.md text) comes from the DB under the DB backend, files + # Source (datasource.md / USER_MEMORY.md text) comes from the DB under the DB backend, files # otherwise — so a DB-only deploy reads no files at runtime. from semantic_model import org_draft as _OD @@ -1361,7 +1361,7 @@ def _record_tool_call(rec: dict[str, Any]) -> None: "budget; `dataset_names=[...]` returns full get_table_context (columns scoped by " "expose_column_groups, default_filters, relationships, caveats, value_transforms, " "metrics) for the named tables; `query` ranks metrics so you don't ingest the whole " - "catalog (`metric_index` lists every metric regardless). Plus ORGANIZATION.md / " + "catalog (`metric_index` lists every metric regardless). Plus datasource.md / " "USER_MEMORY.md context. Use metric `calculation`/`bindings` VERBATIM." ), "inputSchema": { diff --git a/packages/agami-core/src/ui.py b/packages/agami-core/src/ui.py index 51e22ee9..a20ca7f5 100644 --- a/packages/agami-core/src/ui.py +++ b/packages/agami-core/src/ui.py @@ -320,7 +320,7 @@ def _link(m: "re.Match[str]") -> str: def md(text: str) -> str: """A tiny, SAFE markdown subset for the domain-context doc — **escape-first**, so any raw HTML in the source is inert. Supports headings, bold/italic, inline + fenced code, bullet/numbered lists, - and http(s) links. Not a full renderer; just enough for an ORGANIZATION.md.""" + and http(s) links. Not a full renderer; just enough for an datasource.md.""" if not text: return "" out: list[str] = [] diff --git a/plugins/agami/lib/agami_paths.py b/plugins/agami/lib/agami_paths.py index 61ce8714..60ae1719 100644 --- a/plugins/agami/lib/agami_paths.py +++ b/plugins/agami/lib/agami_paths.py @@ -12,7 +12,7 @@ charts/, exports/ # per-query outputs model/, review/, examples-validation/ # rendered dashboards tunnels/, serve/ # ssh tunnels, the copied MCP server - / # the committable semantic model (org.yaml + subject_areas/…) + / # the committable semantic model (datasource.yaml + subject_areas/…) USER_MEMORY.md # committable cross-database preferences .gitignore # ignores local/ diff --git a/plugins/agami/lib/execute_sql.py b/plugins/agami/lib/execute_sql.py index 0b0825a7..fb0cd20a 100644 --- a/plugins/agami/lib/execute_sql.py +++ b/plugins/agami/lib/execute_sql.py @@ -896,7 +896,7 @@ def _resolve_guard_model(profile: str): pass # DB unreachable/misconfigured -> fall through to disk root = Path(os.environ.get("AGAMI_ARTIFACTS_DIR") or (Path.home() / "agami-artifacts")) / profile - if (root / "org.yaml").exists(): + if (root / "datasource.yaml").exists(): try: return L.load_organization(root) except Exception: diff --git a/plugins/agami/lib/semantic_model/__init__.py b/plugins/agami/lib/semantic_model/__init__.py index 78732c5d..55a107fd 100644 --- a/plugins/agami/lib/semantic_model/__init__.py +++ b/plugins/agami/lib/semantic_model/__init__.py @@ -9,7 +9,7 @@ Layout on disk (the canonical profile format, rooted at `//`): - org.yaml # org desc + storage_connections + subject_areas + datasource.yaml # org desc + storage_connections + subject_areas datasources//storage.yaml # physical: storage_type, storage_config subject_areas// subject_area.yaml # desc, default_time_window, tables (TableRefs) diff --git a/plugins/agami/samples/store/model/ORGANIZATION.md b/plugins/agami/samples/store/model/datasource.md similarity index 100% rename from plugins/agami/samples/store/model/ORGANIZATION.md rename to plugins/agami/samples/store/model/datasource.md diff --git a/plugins/agami/samples/store/model/org.yaml b/plugins/agami/samples/store/model/datasource.yaml similarity index 93% rename from plugins/agami/samples/store/model/org.yaml rename to plugins/agami/samples/store/model/datasource.yaml index 1e44c82b..8080a860 100644 --- a/plugins/agami/samples/store/model/org.yaml +++ b/plugins/agami/samples/store/model/datasource.yaml @@ -1,4 +1,4 @@ -organization: agami-example +datasource: agami-example version: 1 description: Acme Store — a synthetic online retailer (sample dataset) with two sides, one-time commerce and recurring subscriptions, for trying agami without connecting a real database. fiscal_year_start_month: 1 diff --git a/plugins/agami/scripts/parse_model_feedback.py b/plugins/agami/scripts/parse_model_feedback.py index 30cdc58c..1835e139 100644 --- a/plugins/agami/scripts/parse_model_feedback.py +++ b/plugins/agami/scripts/parse_model_feedback.py @@ -156,7 +156,7 @@ def _qnames(val: str) -> list[str]: elif jk == "key-terminology": data["key_terminology"] = parsed elif jk == "organization-md": - data["organization_md"] = parsed # JSON-encoded string → the full ORGANIZATION.md text + data["organization_md"] = parsed # JSON-encoded string → the full datasource.md text if bad_targets: needs = {"kind": "malformed_targets", "targets": bad_targets, diff --git a/plugins/agami/scripts/prepare_deploy.py b/plugins/agami/scripts/prepare_deploy.py index e1c27b4b..ad4ebcf3 100644 --- a/plugins/agami/scripts/prepare_deploy.py +++ b/plugins/agami/scripts/prepare_deploy.py @@ -81,7 +81,7 @@ def _widen_one(path: Path) -> None: def _grant_world_read(root: Path) -> None: """`chmod -R a+rX` over the staged model. The deployed container runs as uid 10001, not the operator who owns these files, and mounts them read-only — without this the boot-time model load fails - "Permission denied" on ORGANIZATION.md and the container crash-loops. Only non-secret model files + "Permission denied" on datasource.md and the container crash-loops. Only non-secret model files reach here (`local/` is excluded from staging). Uses `os.walk(followlinks=False)` — NOT `rglob("**")`, which follows directory symlinks on Python @@ -135,7 +135,7 @@ def _merge_env(existing: str, template: str, image_tag: str | None) -> tuple[str def _stage_ignore(artifacts: Path, datasources: list[str] | None): """A `copytree` ignore callable: always drop `local/` (secrets never ship), and — when `datasources` - is given — also drop any TOP-LEVEL profile dir (a dir with an `org.yaml`) not in the chosen set. + is given — also drop any TOP-LEVEL profile dir (a dir with an `datasource.yaml`) not in the chosen set. Install-global files (e.g. `USER_MEMORY.md`) and non-profile entries are always kept. Default (`datasources is None`) stages every model, preserving prior behavior.""" chosen = set(datasources) if datasources else None @@ -147,7 +147,7 @@ def _ignore(dirpath: str, names: list[str]) -> set[str]: if chosen is not None: for name in names: d = artifacts / name - if name not in chosen and d.is_dir() and (d / "org.yaml").is_file(): + if name not in chosen and d.is_dir() and (d / "datasource.yaml").is_file(): drop.add(name) return drop @@ -193,7 +193,7 @@ def prepare(args: argparse.Namespace) -> tuple[str, int]: datasources = getattr(args, "datasources", None) dslist = [s.strip() for s in datasources.split(",") if s.strip()] if datasources else None if dslist: - available = {d.name for d in artifacts.iterdir() if d.is_dir() and (d / "org.yaml").is_file()} + available = {d.name for d in artifacts.iterdir() if d.is_dir() and (d / "datasource.yaml").is_file()} unknown = [d for d in dslist if d not in available] # Fail fast if NONE of the requested datasources exist — a modelless bundle would only break # later at runtime. If SOME are unknown, warn (stderr keeps the stdout status line clean) and @@ -207,7 +207,7 @@ def prepare(args: argparse.Namespace) -> tuple[str, int]: chosen = set(dslist) if staged.exists(): for d in staged.iterdir(): - if d.name in chosen or not (d / "org.yaml").is_file(): + if d.name in chosen or not (d / "datasource.yaml").is_file(): continue # keep chosen models + non-model entries (e.g. USER_MEMORY.md) if d.is_symlink(): d.unlink() # a symlinked model: drop the link, never rmtree its target diff --git a/plugins/agami/scripts/render_model_explorer.py b/plugins/agami/scripts/render_model_explorer.py index b421e666..a6fbe474 100644 --- a/plugins/agami/scripts/render_model_explorer.py +++ b/plugins/agami/scripts/render_model_explorer.py @@ -66,7 +66,7 @@ def build_manifest(profile_dir: Path, profile: str) -> dict: examples_out: list[dict] = [] areas_out: list[dict] = [] - org_md_path = profile_dir / "ORGANIZATION.md" + org_md_path = profile_dir / "datasource.md" organization_md = org_md_path.read_text(encoding="utf-8") if org_md_path.exists() else "" import re as _re @@ -248,7 +248,7 @@ def render(*, title: str, profile: str, manifest: dict) -> str: logo_light = LOGO_LIGHT_PATH.read_text() if LOGO_LIGHT_PATH.exists() else "" theme_css = (SHARED_DIR / "theme.css").read_text() - # The manifest embeds arbitrary model text (descriptions, ORGANIZATION.md, SQL). + # The manifest embeds arbitrary model text (descriptions, datasource.md, SQL). # Escape `` in that text can't terminate the ") + model_store.write_memory(s, "SALES_DATA", datasource_doc="") s.close() _login(client) html = client.get("/admin/model?datasource=SALES_DATA&view=context").text @@ -491,7 +491,7 @@ def test_context_page_empty_when_no_doc(client, env): # --- cross-area (org-level) objects are not dropped -------------------------- CROSS_ORG = { - "organization": "acme", + "datasource": "acme", "storage_connections": [{"name": "c", "storage_type": "PostgreSQL"}], "subject_areas": [{"name": "A", "tables": [], "tables_defined": []}], "cross_subject_area_metrics": [{"name": "global_rev", "calculation": "sum all"}], @@ -510,7 +510,7 @@ def test_cross_area_objects_surface_on_overview(client, env): # --- cross-area relationships view ------------------------------------------- XREL_ORG = { - "organization": "acme", + "datasource": "acme", "storage_connections": [{"name": "c", "storage_type": "PostgreSQL"}], "subject_areas": [ {"name": "finance", "tables": [], "tables_defined": []}, diff --git a/tests/test_aggregation_enforcement.py b/tests/test_aggregation_enforcement.py index 2ffe4711..b54d581e 100644 --- a/tests/test_aggregation_enforcement.py +++ b/tests/test_aggregation_enforcement.py @@ -30,7 +30,7 @@ def _org(columns, metrics=None, table="facts"): columns=columns) sa = m.SubjectArea(name="area", description="d", tables_defined=[t], metrics=metrics or []) - return m.Organization(organization="o", version=1, subject_areas=[sa]) + return m.Organization(datasource="o", version=1, subject_areas=[sa]) # --- #2 aggregation-class enforcement -------------------------------------- @@ -124,7 +124,7 @@ def test_semi_additive_is_table_scoped_no_cross_table_misfire(): bindings={"PostgreSQL": "SUM(balance)"}, source_tables=["facts"], non_additive_dimensions=["time"], semi_additive_agg="last") sa = m.SubjectArea(name="area", description="d", tables_defined=[facts, ledger], metrics=[semi]) - org = m.Organization(organization="o", version=1, subject_areas=[sa]) + org = m.Organization(datasource="o", version=1, subject_areas=[sa]) # ledger.balance is NOT the semi-additive one → allowed r = RT.pre_flight_check("SELECT entry_date, SUM(balance) FROM ledger GROUP BY entry_date", org) assert r.action == "allow" diff --git a/tests/test_cloud_neutral_config.py b/tests/test_cloud_neutral_config.py index a38f003f..1b5f9746 100644 --- a/tests/test_cloud_neutral_config.py +++ b/tests/test_cloud_neutral_config.py @@ -188,7 +188,7 @@ def test_db_backed_serve_needs_no_artifacts_dir(tmp_path, monkeypatch): "main", Organization.model_validate( { - "organization": "acme", + "datasource": "acme", "version": 1, "subject_areas": [ {"name": "sales", "metrics": [{"name": "revenue", "calculation": "sum"}]} diff --git a/tests/test_column_scope_adversarial.py b/tests/test_column_scope_adversarial.py index 1be278a0..dbfd1a40 100644 --- a/tests/test_column_scope_adversarial.py +++ b/tests/test_column_scope_adversarial.py @@ -40,7 +40,7 @@ def _t(name, cols): orders = _t("orders", [("id", "integer"), ("amount", "decimal"), ("customer_id", "integer"), ("status", "string")]) customers = _t("customers", [("id", "integer"), ("name", "string"), ("region", "string")]) - return m.Organization(organization="Shop", + return m.Organization(datasource="Shop", subject_areas=[m.SubjectArea(name="sales", tables_defined=[orders, customers])]) diff --git a/tests/test_column_scope_gate.py b/tests/test_column_scope_gate.py index fc45c91d..3fb221e3 100644 --- a/tests/test_column_scope_gate.py +++ b/tests/test_column_scope_gate.py @@ -35,7 +35,7 @@ def _t(name, cols): orders = _t("orders", [("id", "integer"), ("amount", "decimal"), ("customer_id", "integer"), ("status", "string")]) customers = _t("customers", [("id", "integer"), ("name", "string"), ("region", "string")]) - return m.Organization(organization="Shop", + return m.Organization(datasource="Shop", subject_areas=[m.SubjectArea(name="sales", tables_defined=[orders, customers])]) @@ -154,7 +154,7 @@ def test_case_insensitive_match(): # --- column-scope: degrade-to-allow ---------------------------------------- def test_column_empty_model_allows(): - org = m.Organization(organization="Empty", subject_areas=[m.SubjectArea(name="s")]) + org = m.Organization(datasource="Empty", subject_areas=[m.SubjectArea(name="s")]) assert rt.check_column_scope("SELECT anything FROM whatever", org).action == "allow" diff --git a/tests/test_compose_org_context.py b/tests/test_compose_org_context.py index 2e0587b2..6339aee1 100644 --- a/tests/test_compose_org_context.py +++ b/tests/test_compose_org_context.py @@ -29,7 +29,7 @@ def _org(name: str, area: str, account_means: str) -> Organization: return Organization( - organization=name, + datasource=name, subject_areas=[SubjectArea(name=area, description=f"{area} area")], key_terminology={"Account": account_means}, ) diff --git a/tests/test_coverage_gate_self_evident.py b/tests/test_coverage_gate_self_evident.py index d373282c..ac458587 100644 --- a/tests/test_coverage_gate_self_evident.py +++ b/tests/test_coverage_gate_self_evident.py @@ -34,7 +34,7 @@ def _incident_cols(): def _org(tables): - return m.Organization(organization="o", + return m.Organization(datasource="o", subject_areas=[m.SubjectArea(name="s", tables_defined=tables)]) diff --git a/tests/test_coverage_underenriched.py b/tests/test_coverage_underenriched.py index f1ff8bbb..d6da2ab2 100644 --- a/tests/test_coverage_underenriched.py +++ b/tests/test_coverage_underenriched.py @@ -28,7 +28,7 @@ def _col(name, desc=""): def _org(tables): sa = m.SubjectArea(name="a", description="d", tables_defined=tables) - return m.Organization(organization="o", version=1, subject_areas=[sa]) + return m.Organization(datasource="o", version=1, subject_areas=[sa]) def test_self_evident_blanks_are_fine(): diff --git a/tests/test_derived_metrics.py b/tests/test_derived_metrics.py index 3588c4de..a639d83a 100644 --- a/tests/test_derived_metrics.py +++ b/tests/test_derived_metrics.py @@ -31,7 +31,7 @@ def _metric(name, binding, *, base=None, calc="x", tables=("orders",)): def _org(metrics): sa = m.SubjectArea(name="sales", description="d", metrics=metrics) - return m.Organization(organization="o", version=1, subject_areas=[sa]) + return m.Organization(datasource="o", version=1, subject_areas=[sa]) # --- resolver --------------------------------------------------------------- diff --git a/tests/test_entity_anchor_and_area_desc.py b/tests/test_entity_anchor_and_area_desc.py index 9190089f..0e95a3eb 100644 --- a/tests/test_entity_anchor_and_area_desc.py +++ b/tests/test_entity_anchor_and_area_desc.py @@ -34,8 +34,8 @@ def _area_model(tmp_path): root = tmp_path / "p" (root / "datasources" / "c").mkdir(parents=True) (root / "subject_areas" / "s" / "tables").mkdir(parents=True) - (root / "org.yaml").write_text(yaml.safe_dump({ - "organization": "p", "version": 1, + (root / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "p", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/s"]})) (root / "datasources" / "c" / "storage.yaml").write_text( diff --git a/tests/test_get_datasource_schema_sizing.py b/tests/test_get_datasource_schema_sizing.py index 575eb9f4..f98c0ac0 100644 --- a/tests/test_get_datasource_schema_sizing.py +++ b/tests/test_get_datasource_schema_sizing.py @@ -83,10 +83,10 @@ def _write_model( ) ) area_paths.append(f"subject_areas/{a}") - (root / "org.yaml").write_text( + (root / "datasource.yaml").write_text( yaml.safe_dump( { - "organization": "acme", + "datasource": "acme", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": area_paths, diff --git a/tests/test_guard_context.py b/tests/test_guard_context.py index e3b0434b..ede52a5c 100644 --- a/tests/test_guard_context.py +++ b/tests/test_guard_context.py @@ -41,7 +41,7 @@ def _org(): from_column="customer_id", to_column="id", relationship="many_to_one")] return m.Organization( - organization="Shop", + datasource="Shop", subject_areas=[m.SubjectArea(name="sales", tables_defined=[customers, orders], relationships=rels)], ) diff --git a/tests/test_mcp_harness.py b/tests/test_mcp_harness.py index 109d63c9..ab3538e8 100644 --- a/tests/test_mcp_harness.py +++ b/tests/test_mcp_harness.py @@ -43,7 +43,7 @@ def test_distill_strips_human_scaffolding_for_llm_context(): - # ORGANIZATION.md is injected into the model's prompt; the human-only HTML comment + # datasource.md is injected into the model's prompt; the human-only HTML comment # scaffolding ("auto-generated", "edit freely") is noise the LLM shouldn't see — the # MCP path must strip it to match the query skill's read path. raw = ( @@ -145,8 +145,8 @@ def test_resolve_units_maps_result_columns(monkeypatch, tmp_path): (p / "datasources" / "c").mkdir(parents=True) (p / "subject_areas" / "s" / "tables").mkdir(parents=True) (p / "subject_areas" / "s" / "metrics").mkdir(parents=True) - (p / "org.yaml").write_text(yaml.safe_dump({ - "organization": "p", "version": 1, + (p / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "p", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/s"]})) (p / "datasources" / "c" / "storage.yaml").write_text( @@ -272,8 +272,8 @@ def _write_rich_model(tmp_path): (p / "datasources" / "c").mkdir(parents=True) (p / "subject_areas" / "s" / "tables").mkdir(parents=True) (p / "subject_areas" / "s" / "metrics").mkdir(parents=True) - (p / "org.yaml").write_text(yaml.safe_dump({ - "organization": "p", "version": 1, + (p / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "p", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/s"]})) (p / "datasources" / "c" / "storage.yaml").write_text( diff --git a/tests/test_metadata_sources.py b/tests/test_metadata_sources.py index b40a61b9..e0ed70b7 100644 --- a/tests/test_metadata_sources.py +++ b/tests/test_metadata_sources.py @@ -120,8 +120,8 @@ def test_detect_preset_and_usable_sources(): def _incident_model(root: Path) -> None: (root / "datasources" / "c").mkdir(parents=True) (root / "subject_areas" / "itsm" / "tables").mkdir(parents=True) - (root / "org.yaml").write_text(yaml.safe_dump({ - "organization": "sn", "version": 1, + (root / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "sn", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/itsm"], })) @@ -177,7 +177,7 @@ def tbl(name, extra): tables_defined=[tbl("incident", ["assignment_group"]), tbl("problem", ["assignment_group"])]) sysa = mm.SubjectArea(name="sys", description="d", tables_defined=[tbl("sys_user_group", [])]) - org = mm.Organization(organization="o", version=1, subject_areas=[itsm, sysa]) + org = mm.Organization(datasource="o", version=1, subject_areas=[itsm, sysa]) specs = [{"from_table": t.name, "from_column": c.name, "to_table": decls[c.name.lower()]} for sa in org.subject_areas for t in sa.tables_defined for c in t.columns if c.name.lower() in decls] @@ -195,7 +195,7 @@ def tbl(name): columns=[mm.Column(name="id", type="integer", primary_key=True)]) itsm = mm.SubjectArea(name="itsm", description="d", tables_defined=[tbl("incident"), tbl("problem")]) sysa = mm.SubjectArea(name="sys", description="d", tables_defined=[tbl("sys_user")]) - org = mm.Organization(organization="o", version=1, subject_areas=[itsm, sysa]) + org = mm.Organization(datasource="o", version=1, subject_areas=[itsm, sysa]) specs = [ {"from_table": "incident", "from_column": "problem_id", "to_table": "problem"}, # intra (itsm) {"from_table": "incident", "from_column": "caller_id", "to_table": "sys_user"}, # cross (itsm→sys) @@ -218,8 +218,8 @@ def _servicenow_model(root: Path) -> None: (root / "datasources" / "c").mkdir(parents=True) for area in ("itsm", "sys"): (root / "subject_areas" / area / "tables").mkdir(parents=True) - (root / "org.yaml").write_text(yaml.safe_dump({ - "organization": "sn", "version": 1, + (root / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "sn", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/itsm", "subject_areas/sys"], })) @@ -342,7 +342,7 @@ def tbl(name, cols, perf=None): + [mm.Column(name=c, type="integer") for c in cols]) itsm = mm.SubjectArea(name="itsm", description="d", tables_defined=[tbl("incident", ["caller_id"], big)]) sysa = mm.SubjectArea(name="sys", description="d", tables_defined=[tbl("sys_user", [])]) - org = mm.Organization(organization="o", version=1, subject_areas=[itsm, sysa]) + org = mm.Organization(datasource="o", version=1, subject_areas=[itsm, sysa]) probed = {"n": 0} def runner(sql): diff --git a/tests/test_metric_binding_and_prune.py b/tests/test_metric_binding_and_prune.py index 624faa04..f97f928f 100644 --- a/tests/test_metric_binding_and_prune.py +++ b/tests/test_metric_binding_and_prune.py @@ -23,7 +23,7 @@ def _org(metric): m.Column(name="id", type="integer", primary_key=True), m.Column(name="cost", type="decimal")]) sa = m.SubjectArea(name="s", tables_defined=[t], metrics=[metric]) - return m.Organization(organization="t", subject_areas=[sa]) + return m.Organization(datasource="t", subject_areas=[sa]) # --- binding column check --------------------------------------------------- diff --git a/tests/test_model_deploy.py b/tests/test_model_deploy.py index e1904fcb..1573f2d3 100644 --- a/tests/test_model_deploy.py +++ b/tests/test_model_deploy.py @@ -24,12 +24,12 @@ def _write_model(root: Path, datasource: str, org_name: str = "acme") -> None: - """A minimal but real v2 profile dir: org + one subject area + a table + examples + ORGANIZATION.md.""" + """A minimal but real v2 profile dir: org + one subject area + a table + examples + datasource.md.""" d = root / datasource (d / "subject_areas" / "Catalog" / "tables").mkdir(parents=True, exist_ok=True) (d / "prompt_examples" / "Catalog").mkdir(parents=True, exist_ok=True) - (d / "org.yaml").write_text( - f"organization: {org_name}\nversion: 1\ndescription: A neutral demo model.\n" + (d / "datasource.yaml").write_text( + f"datasource: {org_name}\nversion: 1\ndescription: A neutral demo model.\n" "storage_connections:\n - name: warehouse\n storage_type: PostgreSQL\n" "subject_areas:\n - Catalog\n" ) @@ -43,7 +43,7 @@ def _write_model(root: Path, datasource: str, org_name: str = "acme") -> None: (d / "prompt_examples" / "Catalog" / "examples.yaml").write_text( "examples:\n - question: how many products?\n sql: SELECT COUNT(*) FROM products\n" ) - (d / "ORGANIZATION.md").write_text("# Acme\nNeutral demo domain notes.\n") + (d / "datasource.md").write_text("# Acme\nNeutral demo domain notes.\n") def _store(tmp_path: Path) -> Store: @@ -64,7 +64,7 @@ def test_deploy_loads_model_and_round_trips(tmp_path): org = model_store.load_organization(store, "demo") store.close() assert loaded == ["demo"] - assert org is not None and org.organization == "acme" + assert org is not None and org.datasource == "acme" assert [sa.name for sa in org.subject_areas] == ["Catalog"] assert [t.name for sa in org.subject_areas for t in sa.tables_defined] == ["products"] @@ -90,7 +90,7 @@ def test_examples_memory_and_version_are_loaded(tmp_path): model_deploy.deploy_models(store, arts) examples = store.query("SELECT question FROM prompt_example WHERE datasource = ?", ("demo",)) memory = store.query( - "SELECT content FROM memory WHERE datasource = ? AND kind = 'organization'", ("demo",) + "SELECT content FROM memory WHERE datasource = ? AND kind = 'datasource'", ("demo",) ) version = store.query("SELECT version FROM model_version WHERE datasource = ?", ("demo",)) store.close() @@ -131,7 +131,7 @@ def test_redeploy_clears_removed_examples(tmp_path): def test_main_invalid_yaml_exits_one_not_traceback(tmp_path, monkeypatch): arts = tmp_path / "artifacts" _write_model(arts, "demo") - (arts / "demo" / "org.yaml").write_text("organization: [unterminated\n") # invalid YAML + (arts / "demo" / "datasource.yaml").write_text("datasource: [unterminated\n") # invalid YAML monkeypatch.setenv("AGAMI_DB_URL", "sqlite://" + str(tmp_path / "m.db")) monkeypatch.setenv("AGAMI_ARTIFACTS_DIR", str(arts)) assert model_deploy.main([]) == 1 # caught → clean exit, not an uncaught traceback @@ -141,7 +141,7 @@ def test_multiple_datasources_each_load(tmp_path): arts = tmp_path / "artifacts" _write_model(arts, "demo", org_name="acme") _write_model(arts, "demo2", org_name="beta") - # a non-model subdir (no org.yaml) must be skipped, not error + # a non-model subdir (no datasource.yaml) must be skipped, not error (arts / "local").mkdir(parents=True, exist_ok=True) (arts / "local" / "credentials").write_text("[demo]\nhost=localhost\n") store = _store(tmp_path) @@ -160,7 +160,7 @@ def test_main_deploys_all_then_a_named_datasource(tmp_path, monkeypatch): store = Store.connect("sqlite://" + str(tmp_path / "m.db")) org = model_store.load_organization(store, "demo") store.close() - assert org is not None and org.organization == "acme" + assert org is not None and org.datasource == "acme" def test_main_errors_when_no_model_found(tmp_path, monkeypatch): @@ -205,20 +205,20 @@ def test_main_errors_naming_a_missing_datasource(tmp_path, monkeypatch): arts.mkdir() monkeypatch.setenv("AGAMI_DB_URL", "sqlite://" + str(tmp_path / "m.db")) monkeypatch.setenv("AGAMI_ARTIFACTS_DIR", str(arts)) - rc = model_deploy.main(["nope"]) # no nope/org.yaml + rc = model_deploy.main(["nope"]) # no nope/datasource.yaml assert rc == 1 # --- F14 / ACE-056 + ACE-057: minted org_id stamping + backfill -------------------------------- def test_deploy_stamps_minted_org_id(tmp_path, monkeypatch): - # A model whose org.yaml carries a minted org_id: deploy resolves it (via _default_org -> + # A model whose datasource.yaml carries a minted org_id: deploy resolves it (via _default_org -> # tools.resolved_org_id over the artifacts dir) and stamps serving rows with it, not 'local'. import tools arts = tmp_path / "artifacts" _write_model(arts, "demo") - p = arts / "demo" / "org.yaml" + p = arts / "demo" / "datasource.yaml" p.write_text("org_id: mintedcafe\n" + p.read_text()) monkeypatch.setenv("AGAMI_ARTIFACTS_DIR", str(arts)) monkeypatch.delenv("AGAMI_ORG_ID", raising=False) diff --git a/tests/test_model_snapshots.py b/tests/test_model_snapshots.py index f9b9a1be..171a55df 100644 --- a/tests/test_model_snapshots.py +++ b/tests/test_model_snapshots.py @@ -29,7 +29,7 @@ def _mini_model(root: Path) -> None: (root / "subject_areas" / "a").mkdir(parents=True, exist_ok=True) - (root / "org.yaml").write_text("organization: t\nversion: 1\n", encoding="utf-8") + (root / "datasource.yaml").write_text("datasource: t\nversion: 1\n", encoding="utf-8") (root / "subject_areas" / "a" / "subject_area.yaml").write_text("name: a\n", encoding="utf-8") @@ -39,7 +39,7 @@ def test_hash_is_stable_and_change_sensitive(tmp_path): h1 = S.compute_model_hash(r) assert len(h1) == 12 assert S.compute_model_hash(r) == h1 # stable for identical content - (r / "org.yaml").write_text("organization: t\nversion: 2\n", encoding="utf-8") + (r / "datasource.yaml").write_text("datasource: t\nversion: 2\n", encoding="utf-8") assert S.compute_model_hash(r) != h1 # changes when the model changes @@ -65,14 +65,14 @@ def test_write_snapshot_idempotent(tmp_path): def test_write_snapshot_noop_without_model(tmp_path): - assert S.write_snapshot(tmp_path / "empty") is None # no org.yaml → no snapshot + assert S.write_snapshot(tmp_path / "empty") is None # no datasource.yaml → no snapshot def test_prune_keeps_last_N(tmp_path): r = tmp_path / "prof" _mini_model(r) for i in range(S.KEEP + 5): - (r / "org.yaml").write_text(f"organization: t\nversion: {i}\n", encoding="utf-8") + (r / "datasource.yaml").write_text(f"datasource: t\nversion: {i}\n", encoding="utf-8") S.write_snapshot(r) time.sleep(0.005) # distinct mtimes so prune order is well-defined dirs = [p for p in (r / ".snapshots").iterdir() if p.is_dir()] @@ -89,7 +89,7 @@ def test_model_version_reader_contract(tmp_path, monkeypatch): import tools assert tools._model_version("prof") == h1 time.sleep(0.02) - (r / "org.yaml").write_text("organization: t\nversion: 99\n", encoding="utf-8") + (r / "datasource.yaml").write_text("datasource: t\nversion: 99\n", encoding="utf-8") h2 = S.write_snapshot(r) assert h2 != h1 assert tools._model_version("prof") == h2 diff --git a/tests/test_model_store_roundtrip.py b/tests/test_model_store_roundtrip.py index 4165860f..bcb6484b 100644 --- a/tests/test_model_store_roundtrip.py +++ b/tests/test_model_store_roundtrip.py @@ -21,7 +21,7 @@ from store import Store # noqa: E402 FULL_ORG = { - "organization": "acme", + "datasource": "acme", "version": 1, "description": "Acme Inc.", "fiscal_year_start_month": 4, @@ -142,10 +142,10 @@ def _write_file_model(root): } ) ) - (root / "org.yaml").write_text( + (root / "datasource.yaml").write_text( yaml.safe_dump( { - "organization": "acme", + "datasource": "acme", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/sales"], @@ -194,11 +194,11 @@ def test_memory_and_model_version_round_trip(): s = Store.connect("sqlite://") s.run_migrations() model_store.write_memory( - s, "main", organization="# About\nAcme sells widgets.", user="prefer USD" + s, "main", datasource_doc="# About\nAcme sells widgets.", user="prefer USD" ) model_store.write_model_version(s, "main", "v-abc123", created_at="2026-06-25T00:00:00Z") assert model_store.load_memory(s, "main") == { - "organization": "# About\nAcme sells widgets.", + "datasource": "# About\nAcme sells widgets.", "user": "prefer USD", } assert model_store.newest_model_version(s, "main") == "v-abc123" @@ -212,7 +212,7 @@ def test_tools_serve_memory_and_version_from_db_no_files(tmp_path, monkeypatch): s = Store.connect(db_url) s.run_migrations() model_store.write_memory( - s, "main", organization="# Acme\nWidgets co.", user="exclude test users" + s, "main", datasource_doc="# Acme\nWidgets co.", user="exclude test users" ) model_store.write_model_version(s, "main", "v-deadbeef") s.close() @@ -246,7 +246,7 @@ def test_metric_name_collision_keeps_both_metrics(tmp_path, monkeypatch): tmp_path, monkeypatch, { - "organization": "acme", + "datasource": "acme", "version": 1, "subject_areas": [ {"name": "sales", "metrics": [{"name": "revenue", "calculation": "gross"}]}, @@ -268,7 +268,7 @@ def test_index_floor_sheds_full_metrics_and_flags_truncated(tmp_path, monkeypatc tmp_path, monkeypatch, { - "organization": "acme", + "datasource": "acme", "version": 1, "subject_areas": [{"name": "a", "metrics": metrics}], }, diff --git a/tests/test_multi_area_membership.py b/tests/test_multi_area_membership.py index 6423cacc..d40a8d5c 100644 --- a/tests/test_multi_area_membership.py +++ b/tests/test_multi_area_membership.py @@ -30,7 +30,7 @@ def _two_area_org(): area_b = m.SubjectArea(name="incidents", tables_defined=[own], tables=[ m.TableRef(storage_connection="c", schema="s", table="incident"), m.TableRef(storage_connection="c", schema="s", table="sys_user")]) - return m.Organization(organization="o", subject_areas=[area_a, area_b]) + return m.Organization(datasource="o", subject_areas=[area_a, area_b]) def test_find_table_resolves_referenced_table_from_other_area(): diff --git a/tests/test_org_cache.py b/tests/test_org_cache.py index 78331e4e..4247c82a 100644 --- a/tests/test_org_cache.py +++ b/tests/test_org_cache.py @@ -71,7 +71,7 @@ def fake_load(profile): def test_default_org_id_falls_back_to_local(monkeypatch, tmp_path): - # No env override and no org.yaml anywhere in the artifacts dir -> single-tenant 'local' default. + # No env override and no datasource.yaml anywhere in the artifacts dir -> single-tenant 'local' default. monkeypatch.delenv("AGAMI_ORG_ID", raising=False) monkeypatch.setenv("AGAMI_ARTIFACTS_DIR", str(tmp_path)) # empty -> deployment_org_id() is None tools.resolved_org_id.cache_clear() @@ -80,10 +80,10 @@ def test_default_org_id_falls_back_to_local(monkeypatch, tmp_path): def test_resolved_org_id_reads_minted_uuid_from_org_yaml(monkeypatch, tmp_path): - # F14: with no env override, the id is the uuid minted into a profile's org.yaml (deployment-scoped + # F14: with no env override, the id is the uuid minted into a profile's datasource.yaml (deployment-scoped # scan of the artifacts dir — the active profile need not be named). (tmp_path / "acme").mkdir() - (tmp_path / "acme" / "org.yaml").write_text("org_id: abc123def\norganization: Acme\n") + (tmp_path / "acme" / "datasource.yaml").write_text("org_id: abc123def\norganization: Acme\n") monkeypatch.delenv("AGAMI_ORG_ID", raising=False) monkeypatch.setenv("AGAMI_ARTIFACTS_DIR", str(tmp_path)) tools.resolved_org_id.cache_clear() @@ -93,7 +93,7 @@ def test_resolved_org_id_reads_minted_uuid_from_org_yaml(monkeypatch, tmp_path): def test_env_override_beats_org_yaml(monkeypatch, tmp_path): # AGAMI_ORG_ID always wins (explicit operator/multi-tenant override). (tmp_path / "acme").mkdir() - (tmp_path / "acme" / "org.yaml").write_text("org_id: from-file\norganization: Acme\n") + (tmp_path / "acme" / "datasource.yaml").write_text("org_id: from-file\norganization: Acme\n") monkeypatch.setenv("AGAMI_ARTIFACTS_DIR", str(tmp_path)) monkeypatch.setenv("AGAMI_ORG_ID", "from-env") tools.resolved_org_id.cache_clear() diff --git a/tests/test_org_draft.py b/tests/test_org_draft.py index 34b1b9f7..15ef076d 100644 --- a/tests/test_org_draft.py +++ b/tests/test_org_draft.py @@ -1,4 +1,4 @@ -"""Tests for semantic_model/org_draft.py — the factual ORGANIZATION.md draft, and the +"""Tests for semantic_model/org_draft.py — the factual datasource.md draft, and the render_model_explorer fallback that uses it so the file is never blank.""" from __future__ import annotations @@ -20,8 +20,8 @@ def _model(root): (root / "subject_areas" / "s" / "tables").mkdir(parents=True) (root / "subject_areas" / "s" / "metrics").mkdir(parents=True) (root / "subject_areas" / "s" / "entities").mkdir(parents=True) - (root / "org.yaml").write_text(yaml.safe_dump({ - "organization": "acme", "version": 1, + (root / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "acme", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/s"]})) (root / "datasources" / "c" / "storage.yaml").write_text( @@ -138,18 +138,18 @@ def test_set_key_terminology_merges_then_replaces(tmp_path): def test_explorer_org_md_is_human_only_derived_is_a_separate_field(tmp_path): from render_model_explorer import build_manifest _model(tmp_path) - # no ORGANIZATION.md → editable field is the human STARTER (prompt only, NO facts)... + # no datasource.md → editable field is the human STARTER (prompt only, NO facts)... m = build_manifest(tmp_path, "acme") assert "About this database" in m["organization_md"] assert "Subject areas" not in m["organization_md"] # facts never in the editable file # ...and the model-derived facts live in their own read-only field assert "Subject areas" in m["derived_context_md"] and "sales" in m["derived_context_md"] # a comments-only file is still "blank" → starter; facts stay separate - (tmp_path / "ORGANIZATION.md").write_text("\n") + (tmp_path / "datasource.md").write_text("\n") m2 = build_manifest(tmp_path, "acme") assert "About this database" in m2["organization_md"] and "Subject areas" in m2["derived_context_md"] # a real human file is left as-is in the editable field; facts still separate - (tmp_path / "ORGANIZATION.md").write_text("# About\nWe are a lending startup.") + (tmp_path / "datasource.md").write_text("# About\nWe are a lending startup.") m3 = build_manifest(tmp_path, "acme") assert m3["organization_md"] == "# About\nWe are a lending startup." assert "Subject areas" in m3["derived_context_md"] diff --git a/tests/test_org_identity_e2e.py b/tests/test_org_identity_e2e.py index 85e93e33..2c8ec40d 100644 --- a/tests/test_org_identity_e2e.py +++ b/tests/test_org_identity_e2e.py @@ -27,7 +27,7 @@ def _minimal_org(name: str = "acme") -> Organization: - return Organization(organization=name) + return Organization(datasource=name) def test_write_tree_mints_uuid4_and_is_immutable(tmp_path): @@ -76,28 +76,28 @@ def test_deploy_and_serve_resolve_the_same_id(tmp_path, monkeypatch): def test_ensure_org_id_cli_mints_into_copied_model(tmp_path, capsys): - # The sample-copy path (agami-connect 6A) drops a prebuilt org.yaml with no id; `sm ensure-org-id` + # The sample-copy path (agami-connect 6A) drops a prebuilt datasource.yaml with no id; `sm ensure-org-id` # mints one into it. Idempotent: a second run prints the SAME id and doesn't rewrite. from semantic_model import cli prof = tmp_path / "agami-example" prof.mkdir() - (prof / "org.yaml").write_text("organization: Acme Store\nversion: 1\n") + (prof / "datasource.yaml").write_text("organization: Acme Store\nversion: 1\n") assert cli.main(["ensure-org-id", str(prof)]) == 0 minted = capsys.readouterr().out.strip() assert minted and len(minted) == 32 and set(minted) <= _HEX - assert loader.load_org_id(prof) == minted # persisted into org.yaml + assert loader.load_org_id(prof) == minted # persisted into datasource.yaml assert cli.main(["ensure-org-id", str(prof)]) == 0 assert capsys.readouterr().out.strip() == minted # idempotent — same id, no re-mint def test_legacy_profile_without_org_id_resolves_local(tmp_path, monkeypatch): - # A pre-F14 org.yaml (no org_id key) still loads and resolves to the 'local' sentinel — no crash, + # A pre-F14 datasource.yaml (no org_id key) still loads and resolves to the 'local' sentinel — no crash, # no forced mint at serve time (serve is read-only; minting happens only at connect/build). (tmp_path / "old").mkdir() - (tmp_path / "old" / "org.yaml").write_text("organization: legacy\nversion: 1\n") + (tmp_path / "old" / "datasource.yaml").write_text("organization: legacy\nversion: 1\n") assert loader.load_org_id(tmp_path / "old") is None assert loader.load_organization(tmp_path / "old").org_id is None diff --git a/tests/test_org_record.py b/tests/test_org_record.py index c65911b7..a0bdb5e7 100644 --- a/tests/test_org_record.py +++ b/tests/test_org_record.py @@ -35,7 +35,7 @@ def _minimal_org(name: str = "acme") -> Organization: - return Organization(organization=name) + return Organization(datasource=name) def test_ensure_org_record_mints_once_into_organization_yaml(tmp_path): @@ -72,12 +72,12 @@ def _counting(art): def test_legacy_lift_preserves_a_pre_record_org_id_idempotently(tmp_path): - # A post-F14 / pre-F15 deployment keeps its id in a profile's org.yaml and has no record yet. + # A post-F14 / pre-F15 deployment keeps its id in a profile's datasource.yaml and has no record yet. # ensure_org_record LIFTS that id up (never re-mints), and is idempotent on re-run. prof = tmp_path / "northpeak_salesforce" prof.mkdir() - (prof / "org.yaml").write_text( - yaml.safe_dump({"org_id": "legacyid0000", "organization": "acme"}), encoding="utf-8" + (prof / "datasource.yaml").write_text( + yaml.safe_dump({"org_id": "legacyid0000", "datasource": "acme"}), encoding="utf-8" ) assert OR.load_org_record(tmp_path) is None @@ -97,8 +97,8 @@ def test_resolver_reads_the_record_then_falls_back_to_legacy_then_local(tmp_path # (b) a legacy per-profile id, still no record -> the legacy scan resolves it. prof = tmp_path / "old" prof.mkdir() - (prof / "org.yaml").write_text( - yaml.safe_dump({"org_id": "legacyid0000", "organization": "x"}), encoding="utf-8" + (prof / "datasource.yaml").write_text( + yaml.safe_dump({"org_id": "legacyid0000", "datasource": "x"}), encoding="utf-8" ) tools.resolved_org_id.cache_clear() assert tools.resolved_org_id() == "legacyid0000" @@ -147,7 +147,7 @@ def test_refresh_datasources_tracks_profiles_on_disk(tmp_path): OR.set_org_fields(tmp_path, name="Acme") # a record with authored company content for prof in ("crm", "erp"): (tmp_path / prof).mkdir() - (tmp_path / prof / "org.yaml").write_text(f"org_id: x\norganization: {prof}\n") + (tmp_path / prof / "datasource.yaml").write_text(f"org_id: x\norganization: {prof}\n") rec = OR.refresh_datasources(tmp_path) assert rec.datasources == ["crm", "erp"] # sorted, rebuilt from the profile dirs on disk diff --git a/tests/test_org_record_deploy.py b/tests/test_org_record_deploy.py index ca56df8a..da853e2c 100644 --- a/tests/test_org_record_deploy.py +++ b/tests/test_org_record_deploy.py @@ -117,8 +117,8 @@ def test_coexists_with_a_hosted_shape_row(): def _write_profile(root: Path, datasource: str) -> None: d = root / datasource d.mkdir(parents=True, exist_ok=True) - (d / "org.yaml").write_text( - f"organization: acme\nversion: 1\ndescription: {datasource} model.\n" + (d / "datasource.yaml").write_text( + f"datasource: acme\nversion: 1\ndescription: {datasource} model.\n" "storage_connections:\n - name: warehouse\n storage_type: PostgreSQL\nsubject_areas: []\n" ) @@ -142,7 +142,7 @@ def test_deploy_writes_one_org_row_and_two_datasource_rows_on_one_org_id(tmp_pat org_rows = s.query("SELECT org_id FROM organization") ds_rows = s.query("SELECT org_id, datasource FROM datasource_model ORDER BY datasource") company_mem = s.query( - "SELECT content FROM memory WHERE org_id = ? AND datasource = '' AND kind = 'organization'", + "SELECT content FROM memory WHERE org_id = ? AND datasource = '' AND kind = 'datasource'", ("deployorg",), ) deployed_record = MS.load_organization_record(s, "deployorg") diff --git a/tests/test_organization_context_e2e.py b/tests/test_organization_context_e2e.py index cf03e596..74d848f7 100644 --- a/tests/test_organization_context_e2e.py +++ b/tests/test_organization_context_e2e.py @@ -37,13 +37,13 @@ def _profile(root: Path, name: str, area: str, account_means: str) -> None: org = Organization( - organization=name, + datasource=name, subject_areas=[SubjectArea(name=area, description=f"{area} area")], ) build.write_tree(org, root / name) # key_terminology is written by the enrichment / set-terminology path, not write_tree — inject it so # the source-specific vocabulary ("Account" resolves per source) round-trips through load_organization. - org_yaml = root / name / "org.yaml" + org_yaml = root / name / "datasource.yaml" doc = yaml.safe_load(org_yaml.read_text()) doc["key_terminology"] = {"Account": account_means} org_yaml.write_text(yaml.safe_dump(doc, sort_keys=False)) diff --git a/tests/test_prepare_deploy.py b/tests/test_prepare_deploy.py index e44eafb9..6a70782b 100644 --- a/tests/test_prepare_deploy.py +++ b/tests/test_prepare_deploy.py @@ -32,7 +32,7 @@ def _artifacts(tmp_path: Path) -> Path: creds.write_text("[demo]\nhost = db.example\n", encoding="utf-8") creds.chmod(0o600) (art / "demo").mkdir() - (art / "demo" / "org.yaml").write_text("name: demo\n", encoding="utf-8") + (art / "demo" / "datasource.yaml").write_text("name: demo\n", encoding="utf-8") return art @@ -65,7 +65,7 @@ def test_prepares_a_complete_bundle(tmp_path): assert "build:" not in compose # The MODEL is staged so the bundle is self-contained + shippable... - assert (target / "artifacts" / "demo" / "org.yaml").exists() + assert (target / "artifacts" / "demo" / "datasource.yaml").exists() # ...but no secret travels: `local/` (credentials + .pgpass) is excluded entirely (creds come from # DATASOURCE_URL in agami.env now), so no 600-mode file lands in a shippable bundle or a mounted volume. assert not (target / "artifacts" / "local").exists() @@ -124,13 +124,13 @@ def test_staged_model_is_world_readable(tmp_path): world-traversable and every file world-readable, else the boot-time load crashes (issue #1's fix).""" art = _artifacts(tmp_path) # Simulate a restrictive owner-only source (umask 077); the widening must repair it in the bundle. - (art / "demo" / "org.yaml").chmod(0o600) + (art / "demo" / "datasource.yaml").chmod(0o600) (art / "demo").chmod(0o700) target = tmp_path / "bundle" prepare_deploy.prepare(_args(target, art)) prof = target / "artifacts" / "demo" assert stat.S_IMODE(prof.stat().st_mode) & 0o005 == 0o005 # dir: others r-x (traversable) - assert stat.S_IMODE((prof / "org.yaml").stat().st_mode) & 0o004 # file: others readable + assert stat.S_IMODE((prof / "datasource.yaml").stat().st_mode) & 0o004 # file: others readable def test_widen_does_not_chmod_through_a_dir_symlink(tmp_path): @@ -349,11 +349,11 @@ def test_selective_datasources_stages_only_chosen(tmp_path): """`--datasources` stages only the named models; others drop, install-global files always stay.""" art = _artifacts(tmp_path) # has model `demo` (art / "ops").mkdir() - (art / "ops" / "org.yaml").write_text("name: ops\n", encoding="utf-8") + (art / "ops" / "datasource.yaml").write_text("name: ops\n", encoding="utf-8") (art / "USER_MEMORY.md").write_text("hi\n", encoding="utf-8") target = tmp_path / "bundle" prepare_deploy.prepare(_args(target, art, datasources="demo")) - assert (target / "artifacts" / "demo" / "org.yaml").exists() + assert (target / "artifacts" / "demo" / "datasource.yaml").exists() assert not (target / "artifacts" / "ops").exists() # not chosen → dropped assert (target / "artifacts" / "USER_MEMORY.md").exists() # install-global → always staged @@ -365,7 +365,7 @@ def test_unknown_datasource_name_warns_but_does_not_fail(tmp_path, capsys): target = tmp_path / "bundle" status, code = prepare_deploy.prepare(_args(target, art, datasources="demo,nope")) assert code == 0 and status.startswith("PREPARED ") - assert (target / "artifacts" / "demo" / "org.yaml").exists() + assert (target / "artifacts" / "demo" / "datasource.yaml").exists() assert "nope" in capsys.readouterr().err @@ -374,7 +374,7 @@ def test_rerun_with_datasources_drops_a_previously_staged_model(tmp_path): delete on its own) — else the server would keep serving a model the operator meant to drop.""" art = _artifacts(tmp_path) # model `demo` (art / "ops").mkdir() - (art / "ops" / "org.yaml").write_text("name: ops\n", encoding="utf-8") + (art / "ops" / "datasource.yaml").write_text("name: ops\n", encoding="utf-8") target = tmp_path / "bundle" prepare_deploy.prepare(_args(target, art)) # first run stages all → demo + ops assert (target / "artifacts" / "ops").exists() @@ -389,14 +389,14 @@ def test_rerun_drops_a_symlinked_datasource_without_touching_its_target(tmp_path art = _artifacts(tmp_path) # model `demo` external = tmp_path / "external-ops" external.mkdir() - (external / "org.yaml").write_text("name: ops\n", encoding="utf-8") + (external / "datasource.yaml").write_text("name: ops\n", encoding="utf-8") (art / "ops").symlink_to(external, target_is_directory=True) # a symlinked model in the artifacts dir target = tmp_path / "bundle" prepare_deploy.prepare(_args(target, art)) # stages demo + the ops symlink status, code = prepare_deploy.prepare(_args(target, art, datasources="demo")) # drop ops on re-run assert code == 0 and status.startswith("UPGRADED ") assert not (target / "artifacts" / "ops").exists() # the staged symlink is gone - assert (external / "org.yaml").exists() # its target is untouched + assert (external / "datasource.yaml").exists() # its target is untouched def test_all_unknown_datasources_fails_fast(tmp_path): diff --git a/tests/test_render_model_explorer.py b/tests/test_render_model_explorer.py index f9770b3f..a28d4ed3 100644 --- a/tests/test_render_model_explorer.py +++ b/tests/test_render_model_explorer.py @@ -33,8 +33,8 @@ def profile_dir(tmp_path): (root / "datasources" / "c").mkdir(parents=True) (root / "subject_areas" / "sales" / "tables").mkdir(parents=True) (root / "subject_areas" / "sales" / "metrics").mkdir(parents=True) - (root / "org.yaml").write_text(yaml.safe_dump({ - "organization": "test", "version": 1, + (root / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "test", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/sales"], })) @@ -120,7 +120,7 @@ def test_manifest_surfaces_full_model(profile_dir): (profile_dir / "prompt_examples" / "sales").mkdir(parents=True, exist_ok=True) (profile_dir / "prompt_examples" / "sales" / "examples.yaml").write_text(yaml.safe_dump([ {"question": "how many orders", "sql": "SELECT COUNT(*) FROM orders", "source": "seed"}])) - (profile_dir / "ORGANIZATION.md").write_text("# About\nA shop. MRR = monthly recurring revenue.") + (profile_dir / "datasource.md").write_text("# About\nA shop. MRR = monthly recurring revenue.") (area / "relationships.yaml").write_text(yaml.safe_dump({"relationships": [ {"from_table": "orders", "from_column": "customer_id", "to_table": "customers", "to_column": "id", "relationship": "many_to_one", "review_state": "approved", @@ -159,8 +159,8 @@ def test_manifest_carries_table_description_source_ai_unknown(tmp_path): root = tmp_path / "p" (root / "datasources" / "c").mkdir(parents=True) (root / "subject_areas" / "s" / "tables").mkdir(parents=True) - (root / "org.yaml").write_text(yaml.safe_dump({ - "organization": "p", "version": 1, + (root / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "p", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/s"]})) (root / "datasources" / "c" / "storage.yaml").write_text( diff --git a/tests/test_second_order_metrics.py b/tests/test_second_order_metrics.py index 75788a54..21fc1b96 100644 --- a/tests/test_second_order_metrics.py +++ b/tests/test_second_order_metrics.py @@ -31,7 +31,7 @@ def _metric(name, binding, *, base=None, inner_grain=None, tables=("orders",)): def _org(metrics): sa = m.SubjectArea(name="sales", description="d", metrics=metrics) - return m.Organization(organization="o", version=1, subject_areas=[sa]) + return m.Organization(datasource="o", version=1, subject_areas=[sa]) def test_synthesizes_avg_daily_revenue_cte(): diff --git a/tests/test_semantic_model_cli.py b/tests/test_semantic_model_cli.py index cbab855c..251891db 100644 --- a/tests/test_semantic_model_cli.py +++ b/tests/test_semantic_model_cli.py @@ -26,8 +26,8 @@ def _model(root: Path) -> None: (root / "datasources" / "c").mkdir(parents=True) (root / "subject_areas" / "s" / "tables").mkdir(parents=True) - (root / "org.yaml").write_text(yaml.safe_dump({ - "organization": "p", "version": 1, + (root / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "p", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/s"]})) (root / "datasources" / "c" / "storage.yaml").write_text( @@ -434,7 +434,7 @@ def test_seed_examples_passes_when_columns_described(tmp_path, monkeypatch): def test_no_model_root_exits_3_cleanly(tmp_path): - # an empty root has no org.yaml — the CLI returns a clean no_model signal (exit 3), + # an empty root has no datasource.yaml — the CLI returns a clean no_model signal (exit 3), # not a traceback, so callers fold the existence check into their first real call rc, out = _run(["areas", str(tmp_path)]) assert rc == 3 @@ -656,7 +656,7 @@ def test_column_groups_edit_reconciles_stale_expose(tmp_path): def test_set_terminology_writes_glossary_to_org_yaml(tmp_path): - # the packaged path for the decoded-abbreviation legend: writes org.yaml key_terminology, + # the packaged path for the decoded-abbreviation legend: writes datasource.yaml key_terminology, # validates, merges over existing terms (so a re-run doesn't clobber a human's edits). from semantic_model.loader import load_organization _model(tmp_path) diff --git a/tests/test_semantic_model_curate.py b/tests/test_semantic_model_curate.py index 22b9be54..52dc1e40 100644 --- a/tests/test_semantic_model_curate.py +++ b/tests/test_semantic_model_curate.py @@ -29,8 +29,8 @@ def _write_model(root: Path, *, git: bool = True) -> None: (root / "datasources" / "c").mkdir(parents=True) (root / "subject_areas" / "sales" / "tables").mkdir(parents=True) (root / "subject_areas" / "sales" / "metrics").mkdir(parents=True) - (root / "org.yaml").write_text(yaml.safe_dump({ - "organization": "shop", "version": 1, + (root / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "shop", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/sales"], })) @@ -219,8 +219,8 @@ def test_edit_op_ambiguous_area_errors_clearly(tmp_path): def test_approve_cross_area_relationship_writes_org_yaml(tmp_path): - """A cross-schema/cross-area join lives in org.yaml (not an area's relationships.yaml), - so approving it must update org.yaml via the org-level fallback.""" + """A cross-schema/cross-area join lives in datasource.yaml (not an area's relationships.yaml), + so approving it must update datasource.yaml via the org-level fallback.""" root = tmp_path (root / "datasources" / "c").mkdir(parents=True) (root / "datasources" / "c" / "storage.yaml").write_text( @@ -233,8 +233,8 @@ def test_approve_cross_area_relationship_writes_org_yaml(tmp_path): "name": tbl, "schema": area, "storage_connection": "c", "grain": ["id"], "description": tbl, "columns": [{"name": "id", "type": "integer", "primary_key": True}, {"name": "customer_id", "type": "integer"}]})) - (root / "org.yaml").write_text(yaml.safe_dump({ - "organization": "shop", "version": 1, + (root / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "shop", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/billing", "subject_areas/crm"], "cross_subject_area_relationships": [{ @@ -250,17 +250,17 @@ def test_approve_cross_area_relationship_writes_org_yaml(tmp_path): "name": "invoices->customers", "at": "2026-06-12T00:00:00Z"}], signer="reviewer@example.com", role="data_lead") assert res.validated and res.applied and not res.skipped, res.skipped - o = yaml.safe_load((root / "org.yaml").read_text()) + o = yaml.safe_load((root / "datasource.yaml").read_text()) cr = o["cross_subject_area_relationships"][0] assert cr["review_state"] == "approved" and cr["signed_off_by"] == "reviewer@example.com" - # editing a cross-area join's field also persists to org.yaml (the UI edit path) — + # editing a cross-area join's field also persists to datasource.yaml (the UI edit path) — # not just approve/reject. So cross joins are as editable as regular joins. res2 = curate.apply(root, [{"op": "edit", "kind": "relationship", "area": "billing", "name": "invoices->customers", "field": "description", "value": "invoice to its CRM account"}]) assert res2.validated and res2.applied and not res2.skipped, res2.skipped - cr2 = yaml.safe_load((root / "org.yaml").read_text())["cross_subject_area_relationships"][0] + cr2 = yaml.safe_load((root / "datasource.yaml").read_text())["cross_subject_area_relationships"][0] assert cr2["description"] == "invoice to its CRM account" diff --git a/tests/test_semantic_model_introspect.py b/tests/test_semantic_model_introspect.py index b8c9458c..eba2a332 100644 --- a/tests/test_semantic_model_introspect.py +++ b/tests/test_semantic_model_introspect.py @@ -241,7 +241,7 @@ def test_introspect_writes_canonical_tree_and_loads_back(tmp_path): org, rep = I.introspect("shop", "postgres", runner=_catalog_runner, artifacts_dir=tmp_path, dry_run=False) root = tmp_path / "shop" - assert (root / "org.yaml").exists() + assert (root / "datasource.yaml").exists() reloaded = L.load_organization(root) assert V.validate(reloaded).ok assert {t.name for sa in reloaded.subject_areas for t in sa.tables_defined} == {"customers", "orders"} @@ -256,7 +256,7 @@ def test_legacy_model_backed_up_on_reonboard(tmp_path): I.introspect("shop", "postgres", runner=_catalog_runner, artifacts_dir=tmp_path, dry_run=False) assert (root / ".legacy_backup" / "index.yaml").exists() assert (root / ".legacy_backup" / "PUBLIC" / "_schema.yaml").exists() - assert (root / "org.yaml").exists() # new model written at root + assert (root / "datasource.yaml").exists() # new model written at root # --------------------------------------------------------------------------- diff --git a/tests/test_semantic_model_loader.py b/tests/test_semantic_model_loader.py index 291c359b..12e8c97b 100644 --- a/tests/test_semantic_model_loader.py +++ b/tests/test_semantic_model_loader.py @@ -31,7 +31,7 @@ def _wide_org(): tables=[m.TableRef(storage_connection="c", schema="analytics", table="wide", expose_column_groups=["metrics", "location"])], tables_defined=[t]) - return m.Organization(organization="Acme", + return m.Organization(datasource="Acme", storage_connections=[m.StorageConnection(name="c", storage_type="PostgreSQL")], subject_areas=[sa]) @@ -87,8 +87,8 @@ def test_disk_round_trip(tmp_path): root = tmp_path / ".semantic_v2" (root / "datasources" / "c").mkdir(parents=True) (root / "subject_areas" / "area" / "tables").mkdir(parents=True) - (root / "org.yaml").write_text(yaml.safe_dump({ - "organization": "O", "version": 1, + (root / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "O", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/area"], })) @@ -107,7 +107,7 @@ def test_disk_round_trip(tmp_path): "relationships": [], })) org = L.load_organization(root) - assert org.organization == "O" + assert org.datasource == "O" sa = org.subject_area("area") assert sa is not None and sa.defined_table("t").grain == ["id"] assert org.storage_connection("c").storage_type == "PostgreSQL" diff --git a/tests/test_semantic_model_models.py b/tests/test_semantic_model_models.py index 80ffafa2..c745d692 100644 --- a/tests/test_semantic_model_models.py +++ b/tests/test_semantic_model_models.py @@ -166,7 +166,7 @@ def test_cross_relationship_requires_areas(): def test_org_happy_and_accessors(): t = m.Table(name="t", schema="s", storage_connection="c", grain=["id"], description="d", columns=[_col("id")]) - org = m.Organization(organization="O", + org = m.Organization(datasource="O", storage_connections=[m.StorageConnection(name="c", storage_type="PostgreSQL")], subject_areas=[m.SubjectArea(name="sa", tables_defined=[t])]) assert org.subject_area("sa").defined_table("t") is t @@ -175,4 +175,4 @@ def test_org_happy_and_accessors(): def test_org_bad_fiscal_month_rejected(): with pytest.raises(ValidationError): - m.Organization(organization="O", fiscal_year_start_month=13) + m.Organization(datasource="O", fiscal_year_start_month=13) diff --git a/tests/test_semantic_model_runtime.py b/tests/test_semantic_model_runtime.py index 08f5b36b..fb76daa3 100644 --- a/tests/test_semantic_model_runtime.py +++ b/tests/test_semantic_model_runtime.py @@ -26,7 +26,7 @@ def _sales_org(): m.Relationship(from_table="tickets", to_table="customers", from_column="customer_id", to_column="id", relationship="many_to_one"), ] - return m.Organization(organization="Shop", + return m.Organization(datasource="Shop", subject_areas=[m.SubjectArea(name="sales", relationships=rels)]) @@ -120,7 +120,7 @@ def _entity_org(): maps_to=[m.EntityMapping(table="orders", column="order_no", primary=True)]) o2 = m.Entity(name="Shipment", value_pattern=r"^SH\w+$", maps_to=[m.EntityMapping(table="shipments", column="ship_no", primary=True)]) - return m.Organization(organization="Acme", + return m.Organization(datasource="Acme", subject_areas=[m.SubjectArea(name="b", entities=[o1, o2])]) @@ -168,7 +168,7 @@ def _filter_org(): m.Column(name="total", type="decimal"), m.Column(name="tenant_id", type="integer")], default_filters=["{alias}.deleted_at IS NULL"]) - return m.Organization(organization="S", + return m.Organization(datasource="S", subject_areas=[m.SubjectArea(name="s", tables=[m.TableRef(storage_connection="c", schema="public", table="orders")], tables_defined=[t])]) diff --git a/tests/test_semantic_model_validator_v2.py b/tests/test_semantic_model_validator_v2.py index 94134843..1a145cb9 100644 --- a/tests/test_semantic_model_validator_v2.py +++ b/tests/test_semantic_model_validator_v2.py @@ -25,7 +25,7 @@ def _conn(name="c"): def _org(area, conn="c", **kw): - return m.Organization(organization="O", storage_connections=[_conn(conn)], + return m.Organization(datasource="O", storage_connections=[_conn(conn)], subject_areas=[area], **kw) @@ -84,7 +84,7 @@ def test_table_ref_resolves_org_wide(): tables_defined=[t]) b = m.SubjectArea(name="B", tables=[m.TableRef(storage_connection="c", schema="s", table="shared")], tables_defined=[]) - org = m.Organization(organization="O", storage_connections=[_conn()], subject_areas=[a, b]) + org = m.Organization(datasource="O", storage_connections=[_conn()], subject_areas=[a, b]) res = v.validate(org) assert "orphan_table_ref" not in _codes(res) @@ -284,7 +284,7 @@ def test_cross_area_entity_collision_warns(): tables_defined=[rr], entities=[m.Entity(name="Account", maps_to=[m.EntityMapping(table="rev", column="id", primary=True)])]) - org = m.Organization(organization="O", storage_connections=[_conn()], + org = m.Organization(datasource="O", storage_connections=[_conn()], subject_areas=[area_a, area_b]) res = v.validate(org) assert "cross_area_entity_collision" in _codes(res) @@ -308,7 +308,7 @@ def test_executable_mismatch_on_cross_edge(): edge = m.CrossSubjectAreaRelationship(from_table="a", to_table="b", from_column="x", to_column="y", relationship="many_to_one", executable="same_engine", from_subject_area="A", to_subject_area="B") - org = m.Organization(organization="O", + org = m.Organization(datasource="O", storage_connections=[_conn("c1"), _conn("c2")], subject_areas=[area_a, area_b], cross_subject_area_relationships=[edge]) diff --git a/tests/test_sensitive_gate.py b/tests/test_sensitive_gate.py index 3e232826..511d6dcc 100644 --- a/tests/test_sensitive_gate.py +++ b/tests/test_sensitive_gate.py @@ -20,7 +20,7 @@ def _org(cols): t = m.Table(name="users", schema="public", storage_connection="c", grain=["id"], columns=cols) sa = m.SubjectArea(name="area", description="d", tables_defined=[t]) - return m.Organization(organization="o", version=1, subject_areas=[sa]) + return m.Organization(datasource="o", version=1, subject_areas=[sa]) def test_counts_flagged_pii(): @@ -50,7 +50,7 @@ def test_sensitive_columns_under_excluded_table_not_counted(): columns=[m.Column(name="email", type="string", sensitive=True), m.Column(name="ssn", type="string", sensitive=True)]) sa = m.SubjectArea(name="area", description="d", tables_defined=[t]) - org = m.Organization(organization="o", version=1, subject_areas=[sa]) + org = m.Organization(datasource="o", version=1, subject_areas=[sa]) assert C.sensitive_columns(org)["count"] == 0 diff --git a/tests/test_table_scope_gate.py b/tests/test_table_scope_gate.py index 87fab91d..8f7ea8ee 100644 --- a/tests/test_table_scope_gate.py +++ b/tests/test_table_scope_gate.py @@ -31,7 +31,7 @@ def _scope_org(): def _t(name): return m.Table(name=name, schema="public", storage_connection="c", grain=["id"], description=name, columns=[m.Column(name="id", type="integer")]) - return m.Organization(organization="Shop", + return m.Organization(datasource="Shop", subject_areas=[m.SubjectArea(name="sales", tables_defined=[_t("orders"), _t("customers")])]) @@ -89,7 +89,7 @@ def test_case_insensitive_match(): def test_empty_model_allows(): - org = m.Organization(organization="Empty", subject_areas=[m.SubjectArea(name="s")]) + org = m.Organization(datasource="Empty", subject_areas=[m.SubjectArea(name="s")]) assert rt.check_table_scope("SELECT * FROM anything", org).action == "allow" diff --git a/tests/test_units.py b/tests/test_units.py index 45b6d8d3..664e10c1 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -76,8 +76,8 @@ def _currency_model(root): import yaml (root / "datasources" / "c").mkdir(parents=True) (root / "subject_areas" / "s" / "tables").mkdir(parents=True) - (root / "org.yaml").write_text(yaml.safe_dump({ - "organization": "p", "version": 1, + (root / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "p", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/s"]})) (root / "datasources" / "c" / "storage.yaml").write_text( @@ -160,8 +160,8 @@ def test_resolve_result_units_emits_date_format_token(tmp_path): from semantic_model.loader import load_organization (tmp_path / "datasources" / "c").mkdir(parents=True) (tmp_path / "subject_areas" / "s" / "tables").mkdir(parents=True) - (tmp_path / "org.yaml").write_text(yaml.safe_dump({ - "organization": "p", "version": 1, + (tmp_path / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "p", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/s"]})) (tmp_path / "datasources" / "c" / "storage.yaml").write_text( @@ -190,8 +190,8 @@ def test_unit_round_trips_on_column_and_surfaces_in_context(tmp_path): root = tmp_path (root / "datasources" / "c").mkdir(parents=True) (root / "subject_areas" / "s" / "tables").mkdir(parents=True) - (root / "org.yaml").write_text(yaml.safe_dump({ - "organization": "p", "version": 1, + (root / "datasource.yaml").write_text(yaml.safe_dump({ + "datasource": "p", "version": 1, "storage_connections": [{"name": "c", "ref": "datasources/c/storage.yaml"}], "subject_areas": ["subject_areas/s"]})) (root / "datasources" / "c" / "storage.yaml").write_text( From ab345957972344fd3b44cf5372e8c7c515ede754 Mon Sep 17 00:00:00 2001 From: vishal-agami Date: Sat, 25 Jul 2026 11:34:51 +0530 Subject: [PATCH 2/6] =?UTF-8?q?release:=200.5.0=20=E2=80=94=20datasource?= =?UTF-8?q?=20file=20rename?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump plugin/marketplace/package version to 0.5.0 so the directory-marketplace install lands in a fresh cache dir, and retitle the CHANGELOG section. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude-plugin/marketplace.json | 4 ++-- CHANGELOG.md | 2 +- packages/agami-core/pyproject.toml | 2 +- plugins/agami/.claude-plugin/plugin.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index f9a545c2..552b23d4 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -7,7 +7,7 @@ }, "metadata": { "description": "The trust layer between AI and your data. A governed semantic model for AI-to-database access — every join approved, every metric signed off, every answer with a receipt. Local-first: no infra, no servers, no data leaves your machine.", - "version": "0.4.5", + "version": "0.5.0", "pluginRoot": "./plugins", "tags": ["data", "sql", "governance", "trust-layer", "semantic-model", "local-first", "fair-code"], "repository": "https://github.com/AgamiAI/agami-core", @@ -18,7 +18,7 @@ "name": "agami-core", "source": "./plugins/agami", "description": "A governed trust layer between AI and your database: every join FK-derived or human-approved, every metric signed off, every answer with a receipt (the SQL, the model version, who vouched for each definition). Runs entirely on your machine via Claude's Bash / Read / Write tools — no MCP server or Python required if you have psql/mysql or DuckDB (an optional local MCP server is available for use from Claude Desktop).", - "version": "0.4.5", + "version": "0.5.0", "keywords": ["database", "governance", "trust-layer", "semantic-model", "sql", "postgres", "snowflake"], "category": "data" } diff --git a/CHANGELOG.md b/CHANGELOG.md index d3a9ee3d..7b335e19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ is the source of truth a host installs against — bumping it is what invalidate user's plugin cache (see [CONTRIBUTING.md](CONTRIBUTING.md)). Each released section below corresponds to one such version. -## [Unreleased] +## [0.5.0] — 2026-07-25 ### Changed diff --git a/packages/agami-core/pyproject.toml b/packages/agami-core/pyproject.toml index 977dc420..6a8f2c5e 100644 --- a/packages/agami-core/pyproject.toml +++ b/packages/agami-core/pyproject.toml @@ -11,7 +11,7 @@ build-backend = "setuptools.build_meta" [project] name = "agami-core" -version = "0.4.5" +version = "0.5.0" description = "agami-core — governed semantic model, the shared MCP TOOLS harness, and the unified local query executor." readme = "README.md" requires-python = ">=3.10" diff --git a/plugins/agami/.claude-plugin/plugin.json b/plugins/agami/.claude-plugin/plugin.json index 7bbd4e35..2a8d65bf 100644 --- a/plugins/agami/.claude-plugin/plugin.json +++ b/plugins/agami/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agami-core", "description": "The trust layer between AI and your database, so an agent's answers are defensible. Every join, metric, and named filter is confidence-scored and either FK-derived or human-approved via a review dashboard. Every answer ships a SQL receipt showing what ran and which model version it used. Works across Postgres / MySQL / Snowflake / BigQuery / Redshift / SQL Server / Oracle / Databricks / Trino / DuckDB / SQLite. Local-first: credentials, schema, results never leave your machine.", - "version": "0.4.5", + "version": "0.5.0", "author": { "name": "Agami AI", "email": "skills@agami.ai", From 0bb6167952e515ca99a56d1479553f7e89320226 Mon Sep 17 00:00:00 2001 From: vishal-agami Date: Sat, 25 Jul 2026 12:08:25 +0530 Subject: [PATCH 3/6] fix(core): root company narrative is organization.md, not datasource.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ORGANIZATION.md existed at two levels — the artifacts-dir ROOT (the company narrative, beside organization.yaml) and per-profile. The rename correctly made the per-profile file datasource.md, but the blanket pass also renamed the ROOT company narrative, which must instead be organization.md to pair with organization.yaml and stay distinct from the per-datasource datasource.md. Fixes NARRATIVE_FILENAME (org_record.py, the single source of truth) plus the comments/docs/skill that named the company narrative — agami-connect now writes and detects the company narrative at /organization.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/agami-core/src/semantic_model/cli.py | 2 +- packages/agami-core/src/semantic_model/org_draft.py | 2 +- packages/agami-core/src/semantic_model/org_record.py | 4 ++-- plugins/agami/shared/organization-context-format.md | 4 ++-- plugins/agami/skills/agami-connect/SKILL.md | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/agami-core/src/semantic_model/cli.py b/packages/agami-core/src/semantic_model/cli.py index 3042e2e7..7e086cd7 100644 --- a/packages/agami-core/src/semantic_model/cli.py +++ b/packages/agami-core/src/semantic_model/cli.py @@ -145,7 +145,7 @@ def cmd_org_draft(args) -> int: def cmd_org_context(args) -> int: # The full domain context for the LLM, two-level (F15 / ACE-069): the shared COMPANY block from the - # deployment record (/organization.yaml + the root datasource.md), then this + # deployment record (/organization.yaml + the root organization.md), then this # datasource's source-specific narrative + model-derived summary. With no record it degrades to the # pre-F15 per-profile assembly (byte-identical). This is what the query path injects as context. from . import org_draft diff --git a/packages/agami-core/src/semantic_model/org_draft.py b/packages/agami-core/src/semantic_model/org_draft.py index 2662fa7e..b1743ef7 100644 --- a/packages/agami-core/src/semantic_model/org_draft.py +++ b/packages/agami-core/src/semantic_model/org_draft.py @@ -119,7 +119,7 @@ def compose_context(human_md: str, org: "Organization") -> str: def _company_block(record: "OrgRecord", narrative: str) -> str: - """The shared COMPANY context (F15): name/description + the root datasource.md narrative + + """The shared COMPANY context (F15): name/description + the root organization.md narrative + company-wide display conventions + the company glossary. Rendered ONCE above every datasource.""" lines: list[str] = [f"# {record.name or 'Company'} — company context", ""] if record.description: diff --git a/packages/agami-core/src/semantic_model/org_record.py b/packages/agami-core/src/semantic_model/org_record.py index 8ff1c8b7..986f19eb 100644 --- a/packages/agami-core/src/semantic_model/org_record.py +++ b/packages/agami-core/src/semantic_model/org_record.py @@ -3,7 +3,7 @@ One ``OrgRecord`` lives at ``/organization.yaml`` — ABOVE the per-profile ``//datasource.yaml`` models — and holds the company-wide facts (name, description, fiscal year, display conventions, glossary) that would otherwise be duplicated into every profile's -``datasource.yaml`` and drift. The company narrative lives beside it at ``/datasource.md``. +``datasource.yaml`` and drift. The company narrative lives beside it at ``/organization.md``. This module owns: @@ -29,7 +29,7 @@ # The record and the company narrative both sit at the artifacts-dir ROOT (one deployment = one company), # NOT under a profile dir — that is the whole point: written once, shared by every datasource. RECORD_FILENAME = "organization.yaml" -NARRATIVE_FILENAME = "datasource.md" +NARRATIVE_FILENAME = "organization.md" def record_path(artifacts_dir: str | Path) -> Path: diff --git a/plugins/agami/shared/organization-context-format.md b/plugins/agami/shared/organization-context-format.md index 365fad81..49512c81 100644 --- a/plugins/agami/shared/organization-context-format.md +++ b/plugins/agami/shared/organization-context-format.md @@ -8,14 +8,14 @@ A deployment can connect several databases under **one company**. Company-wide c | Level | Where | Holds | Written | |---|---|---|---| -| **Company** (the deployment) | `/organization.yaml` (the `OrgRecord`) + `/datasource.md` (company narrative) | company `name`/`description`, `fiscal_year_start_month`, `display_conventions` (currency/rounding/week_start), the company-wide `glossary`, and an auto-maintained `datasources` list — plus the company narrative prose | `name`/`description` + narrative at first onboarding; `datasources` rebuilt automatically on each onboard/deploy; conventions/glossary edited via `/agami-model` | +| **Company** (the deployment) | `/organization.yaml` (the `OrgRecord`) + `/organization.md` (company narrative) | company `name`/`description`, `fiscal_year_start_month`, `display_conventions` (currency/rounding/week_start), the company-wide `glossary`, and an auto-maintained `datasources` list — plus the company narrative prose | `name`/`description` + narrative at first onboarding; `datasources` rebuilt automatically on each onboard/deploy; conventions/glossary edited via `/agami-model` | | **Datasource** (each profile) | `//datasource.yaml` + `//datasource.md` | that source's ontology (`key_terminology`, subject areas, …) and a **source-specific** narrative only | per profile | `cli org-context` (local) and `get_datasource_schema` (served) both assemble these two levels: the **company block once**, then each datasource's per-database narrative + derived summary. A federated question spanning several datasources renders the company block **once** and both vocabularies. **With no company record, the output is just the per-database assembly** — no error, nothing to migrate. ### Content-routing rule — where each kind of context goes -- **Company-wide** (fiscal year, company glossary, a display convention true for the whole company, "who we are" prose) → the **company record** (`organization.yaml` + root `datasource.md`). +- **Company-wide** (fiscal year, company glossary, a display convention true for the whole company, "who we are" prose) → the **company record** (`organization.yaml` + root `organization.md`). - **Source-specific** (what THIS database means, a term that resolves differently here) → the **per-profile** files (`//datasource.yaml` `key_terminology` / `//datasource.md`). - **Per-column** units/encodings → the column's `field_metadata` in the structured model — **never** prose. - **Personal / stylistic** (how *I* like results displayed) → `USER_MEMORY.md`. diff --git a/plugins/agami/skills/agami-connect/SKILL.md b/plugins/agami/skills/agami-connect/SKILL.md index 4cf50fdf..dff92d4d 100644 --- a/plugins/agami/skills/agami-connect/SKILL.md +++ b/plugins/agami/skills/agami-connect/SKILL.md @@ -481,15 +481,15 @@ Example to say BEFORE introspecting: *"Found 70 tables across 18 schemas. I'll m Two levels: a **company** description written **once** at the deployment root and shared by every datasource, and a **per-database** narrative for the datasource being connected now. The skill never decides yes/skip for the user; "don't ask clarifying questions" does NOT cancel this — it's required state-gathering. Ask **A** then **B**. -**A. Company context — ask ONCE, on the first onboard into this deployment.** Decide "already authored" by content, not by file existence: treat it as done only if the root `/datasource.md` exists **or** the root `/organization.yaml` already has a non-empty `name`/`description`. If neither, this is the first onboard → ask: +**A. Company context — ask ONCE, on the first onboard into this deployment.** Decide "already authored" by content, not by file existence: treat it as done only if the root `/organization.md` exists **or** the root `/organization.yaml` already has a non-empty `name`/`description`. If neither, this is the first onboard → ask: > What's your **company/organization** called, and a sentence or two on what it does? It's shared across every database you connect here and improves NL→SQL. (e.g. what the company/product is, what "MRR" or "active user" means company-wide.) On an answer, do **BOTH** of these (not one — the record write is what populates `organization.yaml`): 1. **Run this command** to write the company `name` + a short `description` into the record (this is the ONLY thing that fills those fields — a file write alone will not): `bash "$AGAMI_PLUGIN_ROOT/scripts/sm" set-org "" --name "" --description ""` -2. **Write** the fuller paragraph to the **root** `/datasource.md` under `# About this company` (**narrative prose ONLY** — no model facts). +2. **Write** the fuller paragraph to the **root** `/organization.md` under `# About this company` (**narrative prose ONLY** — no model facts). -Then confirm both landed: `/organization.yaml` now shows `name:`/`description:`, and `/datasource.md` exists. +Then confirm both landed: `/organization.yaml` now shows `name:`/`description:`, and `/organization.md` exists. `Skip` → leave the company context empty (the composition simply omits the company block). Mention that richer company context (conventions, glossary) can be added later in `/agami-model`. On a **subsequent** onboard (company context already authored) **do NOT re-ask** — narrate *"Sharing the **``** context you set earlier."* From c201619e105029a4302ced34589a7943ec53e5c1 Mon Sep 17 00:00:00 2001 From: vishal-agami Date: Sat, 25 Jul 2026 17:19:08 +0530 Subject: [PATCH 4/6] =?UTF-8?q?fix(core):=20address=20Copilot=20review=20?= =?UTF-8?q?=E2=80=94=20grammar,=20company-narrative=20doc,=20test-fixture?= =?UTF-8?q?=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up nits from the rename's blanket pass: - Grammar: "an datasource.{md,yaml}" -> "a datasource.{md,yaml}" (article was correct only for the old "an ORGANIZATION.md"). 6 spots across model_deploy.py, models.py, ui.py, agami-save-correction/SKILL.md. - org_draft.py: the shared COMPANY block docstring named the root narrative datasource.md; it is organization.md (datasource.md is the per-profile narrative). - Test fixtures writing datasource.yaml still used the legacy `organization:` field key -> `datasource:` (test_org_cache, test_org_identity_e2e, test_org_record). No behavior change. Includes the two instances Copilot's sampling missed (model_deploy.py grammar, test_org_cache.py fixtures). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/agami-core/src/model_deploy.py | 2 +- packages/agami-core/src/semantic_model/models.py | 2 +- packages/agami-core/src/semantic_model/org_draft.py | 2 +- packages/agami-core/src/ui.py | 2 +- plugins/agami/skills/agami-save-correction/SKILL.md | 6 +++--- tests/test_org_cache.py | 4 ++-- tests/test_org_identity_e2e.py | 4 ++-- tests/test_org_record.py | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/agami-core/src/model_deploy.py b/packages/agami-core/src/model_deploy.py index 22890a9e..64bbfde0 100644 --- a/packages/agami-core/src/model_deploy.py +++ b/packages/agami-core/src/model_deploy.py @@ -187,7 +187,7 @@ def main(argv: list[str] | None = None) -> int: if not loaded: print( f"model_deploy: no model found under {artifacts_dir} " - "(a datasource is a subdir with an datasource.yaml).", + "(a datasource is a subdir with a datasource.yaml).", file=sys.stderr, ) return 1 diff --git a/packages/agami-core/src/semantic_model/models.py b/packages/agami-core/src/semantic_model/models.py index 04d5d67f..55844045 100644 --- a/packages/agami-core/src/semantic_model/models.py +++ b/packages/agami-core/src/semantic_model/models.py @@ -652,7 +652,7 @@ class Organization(_Base): # domain glossary: term -> one-line definition (e.g. "MRR": "monthly recurring revenue"). # Enrichment fills this from decoded abbreviations + choice-field legends; org_draft # renders it into datasource.md's "Key terminology", and it feeds NL→SQL as context. - # The structured home means it survives an datasource.md regeneration (a prose section + # The structured home means it survives a datasource.md regeneration (a prose section # the LLM has to remember to write does not). key_terminology: dict[str, str] = Field(default_factory=dict) diff --git a/packages/agami-core/src/semantic_model/org_draft.py b/packages/agami-core/src/semantic_model/org_draft.py index b1743ef7..69b746d7 100644 --- a/packages/agami-core/src/semantic_model/org_draft.py +++ b/packages/agami-core/src/semantic_model/org_draft.py @@ -174,7 +174,7 @@ def compose_org_context( source_narratives: "list[str] | None" = None, ) -> str: """Two-level org context (F15 / ACE-069). Renders the shared COMPANY block ONCE from the ``OrgRecord`` - (name/description + the root ``datasource.md`` narrative + display conventions + company glossary), + (name/description + the root ``organization.md`` narrative + display conventions + company glossary), then each datasource's model-derived ontology under its own heading. Single-source passes one ontology; a FEDERATED question passes several — the company block still renders exactly once, and each source keeps its own vocabulary (so "Account" resolves per source). diff --git a/packages/agami-core/src/ui.py b/packages/agami-core/src/ui.py index a20ca7f5..8cab831b 100644 --- a/packages/agami-core/src/ui.py +++ b/packages/agami-core/src/ui.py @@ -320,7 +320,7 @@ def _link(m: "re.Match[str]") -> str: def md(text: str) -> str: """A tiny, SAFE markdown subset for the domain-context doc — **escape-first**, so any raw HTML in the source is inert. Supports headings, bold/italic, inline + fenced code, bullet/numbered lists, - and http(s) links. Not a full renderer; just enough for an datasource.md.""" + and http(s) links. Not a full renderer; just enough for a datasource.md.""" if not text: return "" out: list[str] = [] diff --git a/plugins/agami/skills/agami-save-correction/SKILL.md b/plugins/agami/skills/agami-save-correction/SKILL.md index 4adfb3ed..600ce9cc 100644 --- a/plugins/agami/skills/agami-save-correction/SKILL.md +++ b/plugins/agami/skills/agami-save-correction/SKILL.md @@ -163,7 +163,7 @@ Else: is the correction about a BUSINESS TERM specific to this database's domain in Stripe" — what the data fundamentally doesn't include) → org_context. A term → `cli set-terminology` (the structured `key_terminology` glossary). A higher-level narrative ("we don't track refunds…", "who the users - are") → an datasource.md prose line. See the `org_context` edit section for both. + are") → a datasource.md prose line. See the `org_context` edit section for both. → org_context is for ABSTRACT business concepts not tied to one specific column. A correction tied to a specific column belongs in field_metadata, NOT here. Re-check the first rule of the tree before landing here. @@ -177,7 +177,7 @@ Else: pure SQL syntax / typo with no domain knowledge implied 1. **Per-column rule → datasource.md.** "`CUSTOMERS.STATUS` values normalize to Active" is NOT domain context — it's a column-value mapping. Route to `field_metadata` (`choice_field`). 2. **Per-column rule → examples.yaml notes.** This skill never writes to `examples.yaml.notes[]` (that path lives in agami-connect Phase 6d). If you find yourself wanting to write "the order total can be negative" as a note on example #12, route it to `field_metadata` on the actual column instead — the lesson applies to every future query, not just to one example. -3. **Dumping a column-fact into a prose file (or USER_MEMORY).** "Amounts are in INR → show ₹" is a fact about the `amount` column → a `caveat`/`value_transform` on that column (org-wide, structured, in the shared model) — NOT a USER_MEMORY line and NOT an datasource.md prose rule. Route data-facts to the column/table; reserve the prose files for cross-cutting conventions (datasource.md) and personal tics (USER_MEMORY). Don't reflexively ask — classify; ask only when personal-vs-org is genuinely unclear. +3. **Dumping a column-fact into a prose file (or USER_MEMORY).** "Amounts are in INR → show ₹" is a fact about the `amount` column → a `caveat`/`value_transform` on that column (org-wide, structured, in the shared model) — NOT a USER_MEMORY line and NOT a datasource.md prose rule. Route data-facts to the column/table; reserve the prose files for cross-cutting conventions (datasource.md) and personal tics (USER_MEMORY). Don't reflexively ask — classify; ask only when personal-vs-org is genuinely unclear. 4. **Display preference → prose without changing SQL.** If the correction is "always format like X," ALSO modify the seed example's SQL to demonstrate the formatting (so future answers actually apply it, not just describe it). ### Diff-based hints (look at SQL changes for classification clues) @@ -318,7 +318,7 @@ The user's bullet should be self-contained — anyone reading USER_MEMORY.md sho ``` The key is the term; the value is a **self-contained** definition (understandable without the original conversation). It merges by default — existing terms are never lost. **Never** hand-append `- "term" = definition` lines to datasource.md; that's the old prose home and is wrong now. - **A higher-level narrative** — what the data represents, who the users are, what's *not* in this database. Append a sentence/paragraph to `//datasource.md` under `# About this database` (create it with the starter if missing — `cli org-draft "$ROOT" > "$ROOT/datasource.md"`). This file is the human narrative **only** — no `term = definition` lines, no model facts. -- **A cross-cutting display/formatting convention** (a currency symbol or number grouping everyone querying this DB should see) is a *presentation* preference, not domain meaning: route it to `user_preference` → `USER_MEMORY.md`, or — when it's really a fact about one column (units/currency) — to that column's `caveat`/`value_transform`. Do **not** invent an datasource.md "conventions" heading; the file is narrative-only. +- **A cross-cutting display/formatting convention** (a currency symbol or number grouping everyone querying this DB should see) is a *presentation* preference, not domain meaning: route it to `user_preference` → `USER_MEMORY.md`, or — when it's really a fact about one column (units/currency) — to that column's `caveat`/`value_transform`. Do **not** invent a datasource.md "conventions" heading; the file is narrative-only. **Show the user the diff** (Phase 4b) before writing. `set-terminology` is validated (reverts on failure); datasource.md prose is free-form (no validation). diff --git a/tests/test_org_cache.py b/tests/test_org_cache.py index 4247c82a..6809ea97 100644 --- a/tests/test_org_cache.py +++ b/tests/test_org_cache.py @@ -83,7 +83,7 @@ def test_resolved_org_id_reads_minted_uuid_from_org_yaml(monkeypatch, tmp_path): # F14: with no env override, the id is the uuid minted into a profile's datasource.yaml (deployment-scoped # scan of the artifacts dir — the active profile need not be named). (tmp_path / "acme").mkdir() - (tmp_path / "acme" / "datasource.yaml").write_text("org_id: abc123def\norganization: Acme\n") + (tmp_path / "acme" / "datasource.yaml").write_text("org_id: abc123def\ndatasource: Acme\n") monkeypatch.delenv("AGAMI_ORG_ID", raising=False) monkeypatch.setenv("AGAMI_ARTIFACTS_DIR", str(tmp_path)) tools.resolved_org_id.cache_clear() @@ -93,7 +93,7 @@ def test_resolved_org_id_reads_minted_uuid_from_org_yaml(monkeypatch, tmp_path): def test_env_override_beats_org_yaml(monkeypatch, tmp_path): # AGAMI_ORG_ID always wins (explicit operator/multi-tenant override). (tmp_path / "acme").mkdir() - (tmp_path / "acme" / "datasource.yaml").write_text("org_id: from-file\norganization: Acme\n") + (tmp_path / "acme" / "datasource.yaml").write_text("org_id: from-file\ndatasource: Acme\n") monkeypatch.setenv("AGAMI_ARTIFACTS_DIR", str(tmp_path)) monkeypatch.setenv("AGAMI_ORG_ID", "from-env") tools.resolved_org_id.cache_clear() diff --git a/tests/test_org_identity_e2e.py b/tests/test_org_identity_e2e.py index 2c8ec40d..d06e9821 100644 --- a/tests/test_org_identity_e2e.py +++ b/tests/test_org_identity_e2e.py @@ -82,7 +82,7 @@ def test_ensure_org_id_cli_mints_into_copied_model(tmp_path, capsys): prof = tmp_path / "agami-example" prof.mkdir() - (prof / "datasource.yaml").write_text("organization: Acme Store\nversion: 1\n") + (prof / "datasource.yaml").write_text("datasource: Acme Store\nversion: 1\n") assert cli.main(["ensure-org-id", str(prof)]) == 0 minted = capsys.readouterr().out.strip() @@ -97,7 +97,7 @@ def test_legacy_profile_without_org_id_resolves_local(tmp_path, monkeypatch): # A pre-F14 datasource.yaml (no org_id key) still loads and resolves to the 'local' sentinel — no crash, # no forced mint at serve time (serve is read-only; minting happens only at connect/build). (tmp_path / "old").mkdir() - (tmp_path / "old" / "datasource.yaml").write_text("organization: legacy\nversion: 1\n") + (tmp_path / "old" / "datasource.yaml").write_text("datasource: legacy\nversion: 1\n") assert loader.load_org_id(tmp_path / "old") is None assert loader.load_organization(tmp_path / "old").org_id is None diff --git a/tests/test_org_record.py b/tests/test_org_record.py index a0bdb5e7..53ce7f8e 100644 --- a/tests/test_org_record.py +++ b/tests/test_org_record.py @@ -147,7 +147,7 @@ def test_refresh_datasources_tracks_profiles_on_disk(tmp_path): OR.set_org_fields(tmp_path, name="Acme") # a record with authored company content for prof in ("crm", "erp"): (tmp_path / prof).mkdir() - (tmp_path / prof / "datasource.yaml").write_text(f"org_id: x\norganization: {prof}\n") + (tmp_path / prof / "datasource.yaml").write_text(f"org_id: x\ndatasource: {prof}\n") rec = OR.refresh_datasources(tmp_path) assert rec.datasources == ["crm", "erp"] # sorted, rebuilt from the profile dirs on disk From 9e710cd2309d32d92308edc5cae6d90a97893c35 Mon Sep 17 00:00:00 2001 From: vishal-agami Date: Sat, 25 Jul 2026 19:19:37 +0530 Subject: [PATCH 5/6] refactor(core): rename per-datasource model symbols Organization -> Datasource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finishes the datasource rename so the per-datasource *model* symbols match the files/fields, per PR review feedback (the half-state read "org means datasource here but not there"). Concept 1 only — deployment/tenant/company identity is left alone (it is legitimately "org"). - class `Organization` -> `Datasource` - `load_organization`/`write_organization`/`starter_organization_md`/`draft_organization_md` -> `*_datasource` (the `*_record` company functions are untouched) - local/JS vars: `org_md`/`org_doc`/`org_path`/`orgp`/`orgRaw`/`orgEdit` -> datasource/ds forms - model-explorer tab `id:'organization'`/`renderOrganizationTab` -> `datasource`/`renderDatasourceTab`, and the `organization_md` manifest + `organization-md` back-channel key -> `datasource_md`/`datasource-md` (render script + template moved in lockstep so the tab still renders + narrative edits still parse) - fix stale org_id-source comments: the id resolves from organization.yaml (F15), not datasource.yaml KEEP (unchanged): OrgRecord, org_id, resolved_org_id, SingleTenantOrgResolver, load/write_organization_record, load_org_record, organization.yaml/.md. Verified: ruff + 1557 tests + gitleaks + lib-drift green (hermetic); explorer re-renders with the Datasource tab (0 old tokens); real model loads as Datasource; feedback round-trips datasource-md; validator clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- dev/render_previews.py | 10 +-- docs/format-spec.md | 2 +- docs/privacy.md | 2 +- packages/agami-core/src/admin.py | 10 +-- packages/agami-core/src/execute_sql.py | 6 +- packages/agami-core/src/mcp_http.py | 6 +- .../src/migrations/core/001_serving.sql | 8 +- packages/agami-core/src/model_deploy.py | 8 +- packages/agami-core/src/model_store.py | 38 ++++----- .../agami-core/src/semantic_model/__init__.py | 2 +- .../agami-core/src/semantic_model/build.py | 4 +- packages/agami-core/src/semantic_model/cli.py | 56 ++++++------- .../agami-core/src/semantic_model/curate.py | 52 ++++++------- .../src/semantic_model/introspect.py | 10 +-- .../agami-core/src/semantic_model/loader.py | 78 +++++++++---------- .../agami-core/src/semantic_model/models.py | 16 ++-- .../src/semantic_model/org_draft.py | 20 ++--- .../agami-core/src/semantic_model/runtime.py | 44 +++++------ .../src/semantic_model/validator.py | 26 +++---- packages/agami-core/src/tools.py | 10 +-- plugins/agami/lib/execute_sql.py | 6 +- plugins/agami/lib/semantic_model/__init__.py | 2 +- plugins/agami/scripts/parse_model_feedback.py | 10 +-- .../agami/scripts/render_model_explorer.py | 16 ++-- .../agami/shared/model-explorer-template.html | 50 ++++++------ .../shared/organization-context-format.md | 2 +- plugins/agami/shared/sql-generation-rules.md | 2 +- plugins/agami/skills/agami-connect/SKILL.md | 2 +- plugins/agami/skills/agami-model/SKILL.md | 8 +- plugins/agami/skills/agami-query/SKILL.md | 6 +- tests/test_ace046_incremental_curation.py | 4 +- tests/test_ace047_schema_index.py | 6 +- tests/test_ace051_fail_closed.py | 6 +- tests/test_admin_model.py | 6 +- tests/test_aggregation_class.py | 2 +- tests/test_aggregation_enforcement.py | 4 +- tests/test_choice_field_decode.py | 4 +- tests/test_cloud_neutral_config.py | 6 +- tests/test_column_scope_adversarial.py | 2 +- tests/test_column_scope_gate.py | 4 +- tests/test_compose_org_context.py | 6 +- tests/test_coverage_gate_self_evident.py | 2 +- tests/test_coverage_underenriched.py | 2 +- tests/test_derived_metrics.py | 2 +- tests/test_guard_context.py | 2 +- tests/test_mcp_harness.py | 2 +- tests/test_metadata_sources.py | 12 +-- tests/test_metric_binding_and_prune.py | 2 +- tests/test_model_deploy.py | 6 +- tests/test_model_feedback.py | 4 +- tests/test_model_store_roundtrip.py | 34 ++++---- tests/test_multi_area_membership.py | 2 +- tests/test_org_cache.py | 2 +- tests/test_org_draft.py | 32 ++++---- tests/test_org_identity_e2e.py | 10 +-- tests/test_org_record.py | 8 +- tests/test_organization_context_e2e.py | 6 +- tests/test_render_model_explorer.py | 2 +- tests/test_sample_database.py | 20 ++--- tests/test_second_order_metrics.py | 2 +- tests/test_semantic_model_cli.py | 46 +++++------ tests/test_semantic_model_curate.py | 22 +++--- tests/test_semantic_model_introspect.py | 8 +- tests/test_semantic_model_loader.py | 4 +- tests/test_semantic_model_models.py | 6 +- tests/test_semantic_model_runtime.py | 6 +- tests/test_semantic_model_validator_v2.py | 10 +-- tests/test_sensitive_gate.py | 4 +- tests/test_table_scope_gate.py | 4 +- tests/test_units.py | 16 ++-- 70 files changed, 420 insertions(+), 420 deletions(-) diff --git a/dev/render_previews.py b/dev/render_previews.py index 43151775..4cde1549 100644 --- a/dev/render_previews.py +++ b/dev/render_previews.py @@ -124,8 +124,8 @@ def write(name: str, html: str) -> None: write("07-admin-activity.html", admin.activity_tab_html(model_store.list_sessions(_s), **CHROME)) # The read-only model explorer — rendered from the REAL builders over the SAME served tree the MCP -# tools read (load_organization), so the previews can't drift from production. Neutral demo data only. -from semantic_model.models import Organization # noqa: E402 +# tools read (load_datasource), so the previews can't drift from production. Neutral demo data only. +from semantic_model.models import Datasource # noqa: E402 _MODEL_ORG = { "organization": "acme", @@ -255,12 +255,12 @@ def write(name: str, html: str) -> None: "- **Net revenue** excludes cancelled orders and nets refunds.\n" "- A **member** has a row in the loyalty table — not just anyone who checked out.\n" ) -model_store.write_organization(_s, "SALES_DATA", Organization.model_validate(_MODEL_ORG)) -model_store.write_organization(_s, "MARKETING", Organization.model_validate(_MODEL_ORG)) # → picker +model_store.write_datasource(_s, "SALES_DATA", Datasource.model_validate(_MODEL_ORG)) +model_store.write_datasource(_s, "MARKETING", Datasource.model_validate(_MODEL_ORG)) # → picker model_store.write_memory(_s, "SALES_DATA", organization=_ORG_MD) model_store.write_model_version(_s, "SALES_DATA", "a1f4c39c0b2e", created_at="2026-06-27T09:00:00Z") _dss = model_store.list_datasources(_s) -_org = model_store.load_organization(_s, "SALES_DATA") +_org = model_store.load_datasource(_s, "SALES_DATA") _ver = model_store.newest_model_version(_s, "SALES_DATA") _catalog = next(a for a in _org.subject_areas if a.name == "Catalog") _products = next(t for t in _catalog.tables_defined if t.name == "products") diff --git a/docs/format-spec.md b/docs/format-spec.md index aa915205..b9895993 100644 --- a/docs/format-spec.md +++ b/docs/format-spec.md @@ -70,7 +70,7 @@ password = mypassword The model is a **provider-portable, standard-concepts hierarchy** that any LLM can traverse to build reliable SQL against any backend: ``` -Organization (datasource.yaml) +Datasource (datasource.yaml) ├─ Storage Connections[] (physical: host/creds/dialect — datasources//storage.yaml) └─ Subject Areas[] (logical — the primary unit the LLM consumes; cap ~20-30 tables) ├─ tables[] (TableRefs into storage connections; expose_column_groups scopes wide tables) diff --git a/docs/privacy.md b/docs/privacy.md index a09b0c2b..d15e4274 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -45,7 +45,7 @@ Every byte agami reads or writes stays on your machine: - **Config** (`/local/.config`) — `active_profile`, `tool_paths`, `reviewer_email`, `reviewer_role` (the artifacts-dir location lives in the `~/.config/agami/path` pointer) - **Semantic model** (`datasource.yaml` + the `subject_areas//` tree under `//`; default `` is `~/agami-artifacts/`) - **Examples library** (`//examples.yaml`) -- **Organization context** (`//datasource.md`) — your description of what the database represents, domain terminology +- **Datasource context** (`//datasource.md`) — your description of what the database represents, domain terminology - **User memory** (`/USER_MEMORY.md`) — your cross-database preferences - **Query results** (everything the assistant shows you) - **Query log** (`/local/query_log.jsonl`) — your personal record of every query you ran diff --git a/packages/agami-core/src/admin.py b/packages/agami-core/src/admin.py index 519d8816..39838502 100644 --- a/packages/agami-core/src/admin.py +++ b/packages/agami-core/src/admin.py @@ -737,7 +737,7 @@ async def admin_set_status(request: Request) -> Response: # --------------------------------------------------------------------------- # Admin console — Model tab (read-only model explorer) # -# A pure projection of the served model (`model_store.load_organization`) + the domain docs +# A pure projection of the served model (`model_store.load_datasource`) + the domain docs # (`load_memory`) — the SAME tree every MCP tool reads, so there is zero drift and no second store. # Read-only by construction: the only route is a GET (see `routes()`), there is no write path, and # `storage_connections[].storage_config` (hosts/credentials) is NEVER rendered. The catalog idiom (a @@ -1254,10 +1254,10 @@ def model_context_html( ) -> str: """The Domain-context page — the deployed datasource.md rendered as (safe) markdown.""" tree = _model_tree_html(org, datasource, datasources, active_view="context") - org_md = memory.get("datasource") + datasource_md = memory.get("datasource") doc = ( - f'
{ui.md(org_md)}
' - if org_md + f'
{ui.md(datasource_md)}
' + if datasource_md else '

No domain context (datasource.md) deployed for this datasource.

' ) content = ( @@ -1291,7 +1291,7 @@ async def admin_model(request: Request) -> Response: datasource = request.query_params.get("datasource") or datasources[0] if datasource not in datasources: # an unknown/stale datasource param → the first served datasource = datasources[0] - org = model_store.load_organization(store, datasource, org_id=org_id) + org = model_store.load_datasource(store, datasource, org_id=org_id) if org is None: return HTMLResponse(model_empty_html(datasource, datasources, **chrome)) view = request.query_params.get("view") diff --git a/packages/agami-core/src/execute_sql.py b/packages/agami-core/src/execute_sql.py index fb0cd20a..865e3a9c 100644 --- a/packages/agami-core/src/execute_sql.py +++ b/packages/agami-core/src/execute_sql.py @@ -867,7 +867,7 @@ def _hosted() -> bool: def _resolve_guard_model(profile: str): """Resolve the semantic model for the safety pass, mirroring `tools._load_org` (ACE-051): from the DB when one is configured (hosted — the `/artifacts` disk mount may be absent), else the - on-disk YAML (local). Returns an `Organization` or None if neither is available. + on-disk YAML (local). Returns an `Datasource` or None if neither is available. The DB import is lazy AND env-guarded on purpose: the local executor runs from a stdlib-lean mirror that does not ship `store`/`model_store`, so we only reach for them when a DB is set. @@ -881,7 +881,7 @@ def _resolve_guard_model(profile: str): # itself, not a diagnostic line. if _hosted(): try: - from model_store import load_organization as _load_db + from model_store import load_datasource as _load_db from store import Store store = Store.from_env() @@ -898,7 +898,7 @@ def _resolve_guard_model(profile: str): root = Path(os.environ.get("AGAMI_ARTIFACTS_DIR") or (Path.home() / "agami-artifacts")) / profile if (root / "datasource.yaml").exists(): try: - return L.load_organization(root) + return L.load_datasource(root) except Exception: pass # unparseable/absent on disk -> None (hosted then fails closed) return None diff --git a/packages/agami-core/src/mcp_http.py b/packages/agami-core/src/mcp_http.py index a3b21511..9cff59a8 100644 --- a/packages/agami-core/src/mcp_http.py +++ b/packages/agami-core/src/mcp_http.py @@ -106,15 +106,15 @@ def _build_auth_provider() -> AuthProvider: def _build_org_resolver() -> SingleTenantOrgResolver: """The OSS default tenancy: single-tenant, one configured org. The id is the F14 minted uuid - resolved by `tools.resolved_org_id()` (AGAMI_ORG_ID env -> datasource.yaml -> "local") — the SAME + resolved by `tools.resolved_org_id()` (AGAMI_ORG_ID env -> organization.yaml -> "local") — the SAME resolution the deploy stamp uses, so writes and reads agree. Multi-tenant is a future change at the *schema* layer plus an authz check — not a resolver swap, so the seam lives here now.""" org_id = resolved_org_id() # Load-bearing for a hand-rolled DB-only deploy: log the resolved id + where it came from, so an - # operator can see "org_id=local (default)" and realize datasource.yaml/AGAMI_ORG_ID isn't reaching the + # operator can see "org_id=local (default)" and realize organization.yaml/AGAMI_ORG_ID isn't reaching the # server (see F14's documented residual risk). source = "env" if os.environ.get("AGAMI_ORG_ID", "").strip() else ( - "default" if org_id == "local" else "datasource.yaml" + "default" if org_id == "local" else "organization.yaml" ) _log.info("single-tenant org_id=%s (source=%s)", org_id, source) return SingleTenantOrgResolver(Org(id=org_id)) diff --git a/packages/agami-core/src/migrations/core/001_serving.sql b/packages/agami-core/src/migrations/core/001_serving.sql index 674f8307..ac7b2524 100644 --- a/packages/agami-core/src/migrations/core/001_serving.sql +++ b/packages/agami-core/src/migrations/core/001_serving.sql @@ -1,6 +1,6 @@ -- agami-core serving schema — the semantic model, served from the DB instead of local YAML files. -- --- Shape mirrors the in-memory Organization tree (semantic_model/models.py) under an +-- Shape mirrors the in-memory Datasource tree (semantic_model/models.py) under an -- org -> datasource -> subject-area hierarchy. Each object row carries the structural/key columns -- a tool queries on PLUS a `doc` (the object's JSON) so the loader rebuilds the EXACT pydantic -- object losslessly — no per-field column sprawl, and YAML stays the source of truth. @@ -17,10 +17,10 @@ -- model_version -> the execute_sql trust receipt's version pin -- (list_datasources reads the set of `datasource` rows; execute_sql/log_feedback write runtime rows — 002) --- The root of a datasource's served model (the pydantic Organization: version, glossary, fiscal year, +-- The root of a datasource's served model (the pydantic Datasource: version, glossary, fiscal year, -- storage_connections, cross-subject-area objects). Named `datasource_model`, not `organization` — -- `org_id` is just the scoping key here; any registry it points at lives outside this table. This is --- the MODEL, one row per (org, datasource). model_store.load_organization / write_organization read/ +-- the MODEL, one row per (org, datasource). model_store.load_datasource / write_datasource read/ -- write it (their names track the pydantic type, not the table). CREATE TABLE datasource_model ( org_id TEXT NOT NULL DEFAULT 'local', @@ -77,7 +77,7 @@ CREATE TABLE entity ( ); -- WITHIN-subject-area relationships only. Cross-subject-area / cross-datasource relationships are --- org-level (Organization.cross_subject_area_relationships) and ride inside datasource_model.doc — they +-- org-level (Datasource.cross_subject_area_relationships) and ride inside datasource_model.doc — they -- are rebuilt from there, not stored as rows here (which would need a null area, see the NOTE above). CREATE TABLE relationship ( org_id TEXT NOT NULL DEFAULT 'local', diff --git a/packages/agami-core/src/model_deploy.py b/packages/agami-core/src/model_deploy.py index 64bbfde0..9019548d 100644 --- a/packages/agami-core/src/model_deploy.py +++ b/packages/agami-core/src/model_deploy.py @@ -21,7 +21,7 @@ def _default_org() -> str: """The org to deploy under when the caller names none. A CLI has no request, so it calls the SAME resolver the server's read path uses (`tools.resolved_org_id`: AGAMI_ORG_ID -> the minted uuid in - datasource.yaml -> 'local') — the two MUST agree, or the model is written under one org and read under + organization.yaml -> 'local') — the two MUST agree, or the model is written under one org and read under another and the server sees no model (F14 / ACE-056).""" from tools import resolved_org_id # lazy: keeps the deploy CLI's import surface small @@ -43,7 +43,7 @@ def _backfill_org_id(store: Store, org_id: str) -> None: (F14 / ACE-057). Runs once at boot, right after migrations, so an EXISTING deployment that ran under 'local' before this feature adopts its minted id instead of orphaning those rows. - Why an UPDATE-move and not a re-seed: `model_store.write_organization`'s redeploy DELETE is scoped + Why an UPDATE-move and not a re-seed: `model_store.write_datasource`'s redeploy DELETE is scoped to (org_id, datasource), so re-deploying under a NEW org_id would leave the old 'local' serving rows behind (doubled); the runtime tables are append-only and can only be corrected by an UPDATE. Idempotent + safe: a no-op when the target is still 'local' (a pre-F14 / un-minted deployment), and @@ -71,7 +71,7 @@ def deploy_one(store: Store, datasource: str, profile_dir: Path, org_id: str | N org_id = org_id if org_id is not None else _default_org() # --- read + parse everything first (where malformed input fails, before any write) --- - org = loader.load_organization(profile_dir) + org = loader.load_datasource(profile_dir) # Examples live per subject area (prompt_examples//examples.yaml); tag each with its area so the # served row carries it (write_examples reads ex["area"]). A malformed examples file for one area is # skipped with a warning, not fatal — examples are best-effort few-shots, not the model itself, and a bad @@ -89,7 +89,7 @@ def deploy_one(store: Store, datasource: str, profile_dir: Path, org_id: str | N version = newest_version(profile_dir) or "deployed" # --- then write (version last, so its presence marks a completed deploy) --- - model_store.write_organization(store, datasource, org, org_id=org_id) + model_store.write_datasource(store, datasource, org, org_id=org_id) # Always write examples (even []) so a redeploy after REMOVING examples actually clears the stale rows — # write_examples is clear-then-insert, so an empty list replaces the datasource's examples with none. model_store.write_examples(store, datasource, examples, org_id=org_id) diff --git a/packages/agami-core/src/model_store.py b/packages/agami-core/src/model_store.py index a6db2bca..71a6a96a 100644 --- a/packages/agami-core/src/model_store.py +++ b/packages/agami-core/src/model_store.py @@ -1,10 +1,10 @@ """Serve the semantic model from the DB — the read path + the deploy-time writer. -The model-loader seam is just "produce an `Organization`": the file adapter is -`semantic_model.loader.load_organization(root)`; this is the DB adapter, which rebuilds the -**identical** `Organization` from rows so every downstream tool (get_datasource_schema incl. -sizing, the receipt) is untouched. YAML stays the source of truth — `write_organization` seeds the -rows from a YAML-loaded `Organization` at deploy time (the `deploy_semantic_model.py` path). +The model-loader seam is just "produce an `Datasource`": the file adapter is +`semantic_model.loader.load_datasource(root)`; this is the DB adapter, which rebuilds the +**identical** `Datasource` from rows so every downstream tool (get_datasource_schema incl. +sizing, the receipt) is untouched. YAML stays the source of truth — `write_datasource` seeds the +rows from a YAML-loaded `Datasource` at deploy time (the `deploy_semantic_model.py` path). Each object is stored as its key/structural columns + a `doc` (the object's `model_dump`), so the rebuild is lossless without enumerating every pydantic field. The parent docs exclude their child @@ -18,10 +18,10 @@ from typing import Any from uuid import uuid4 -from semantic_model.models import Organization, OrgRecord +from semantic_model.models import Datasource, OrgRecord from store import Store -# The per-datasource model tables write_organization clears before a re-seed (so a redeploy +# The per-datasource model tables write_datasource clears before a re-seed (so a redeploy # reproduces the served model rather than appending duplicates / hitting PK conflicts). Must stay in # sync with migrations/core/001_serving.sql's serving tables; examples/memory/model_version are # re-seeded by their own writers, so they're not in this list. @@ -46,10 +46,10 @@ def _est_rows(table_doc: dict[str, Any]) -> int | None: return ph.get("estimated_row_count") if isinstance(ph, dict) else None -def write_organization( - store: Store, datasource: str, org: Organization, org_id: str = DEFAULT_ORG +def write_datasource( + store: Store, datasource: str, org: Datasource, org_id: str = DEFAULT_ORG ) -> None: - """(Re)seed the serving rows for `datasource` from a loaded Organization. Idempotent — clears + """(Re)seed the serving rows for `datasource` from a loaded Datasource. Idempotent — clears the datasource's existing model rows first, so re-running the deploy reproduces the served model.""" for tbl in _MODEL_TABLES: # Scoped by org as well as datasource. Without the org predicate one tenant's redeploy would @@ -58,10 +58,10 @@ def write_organization( f"DELETE FROM {tbl} WHERE org_id = ? AND datasource = ?", (org_id, datasource) ) - org_doc = org.model_dump(mode="json", exclude={"subject_areas"}) + ds_doc = org.model_dump(mode="json", exclude={"subject_areas"}) store.execute( "INSERT INTO datasource_model (org_id, datasource, description, doc) VALUES (?, ?, ?, ?)", - (org_id, datasource, org.description or None, json.dumps(org_doc)), + (org_id, datasource, org.description or None, json.dumps(ds_doc)), ) for sa in org.subject_areas: @@ -111,16 +111,16 @@ def write_organization( store.commit() -def load_organization( +def load_datasource( store: Store, datasource: str, org_id: str = DEFAULT_ORG -) -> Organization | None: - """Rebuild the Organization for `datasource` from rows, or None if it isn't seeded.""" +) -> Datasource | None: + """Rebuild the Datasource for `datasource` from rows, or None if it isn't seeded.""" org_rows = store.query( "SELECT doc FROM datasource_model WHERE org_id = ? AND datasource = ?", (org_id, datasource) ) if not org_rows: return None - org_doc: dict[str, Any] = json.loads(org_rows[0]["doc"]) + ds_doc: dict[str, Any] = json.loads(org_rows[0]["doc"]) subject_areas = [] for sa_row in store.query( @@ -145,8 +145,8 @@ def load_organization( ] subject_areas.append(sa_doc) - org_doc["subject_areas"] = subject_areas - return Organization.model_validate(org_doc) + ds_doc["subject_areas"] = subject_areas + return Datasource.model_validate(ds_doc) def list_datasources(store: Store, org_id: str = DEFAULT_ORG) -> list[str]: @@ -161,7 +161,7 @@ def list_datasources(store: Store, org_id: str = DEFAULT_ORG) -> list[str]: def model_table_counts(store: Store, org_id: str = DEFAULT_ORG) -> dict[str, int]: """`{datasource: table_count}` for the org's served datasources, in ONE grouped query — so the datasource listing sizes itself without a per-datasource round trip (no N+1) and without - rebuilding the whole Organization. A datasource with no modeled tables simply won't appear in + rebuilding the whole Datasource. A datasource with no modeled tables simply won't appear in the map; the caller defaults it to 0.""" rows = store.query( "SELECT datasource, count(*) AS n FROM model_table WHERE org_id = ? GROUP BY datasource", diff --git a/packages/agami-core/src/semantic_model/__init__.py b/packages/agami-core/src/semantic_model/__init__.py index 55a107fd..f3524a4a 100644 --- a/packages/agami-core/src/semantic_model/__init__.py +++ b/packages/agami-core/src/semantic_model/__init__.py @@ -1,6 +1,6 @@ """agami semantic model — provider-portable, standard-concepts hierarchy. -This is **the** agami semantic model: a hierarchy of Organization → Storage +This is **the** agami semantic model: a hierarchy of Datasource → Storage Connection (physical) + Subject Area (logical) → Table / Entity / Metric / Relationship, with provider-portable declarative fields (default_filters, value_transform, caveats, value_pattern, sensitive, cardinality, …) so that any diff --git a/packages/agami-core/src/semantic_model/build.py b/packages/agami-core/src/semantic_model/build.py index cf499ec2..220770fb 100644 --- a/packages/agami-core/src/semantic_model/build.py +++ b/packages/agami-core/src/semantic_model/build.py @@ -21,7 +21,7 @@ DEEP_TABLE_COLUMN_THRESHOLD, Column, CrossSubjectAreaRelationship, - Organization, + Datasource, Relationship, SubjectArea, Table, @@ -505,7 +505,7 @@ def ensure_org_id(out: Path, existing: Optional[str] = None, *, dry_run: bool = def write_tree( - org: Organization, + org: Datasource, out: Path, *, examples_by_area: Optional[dict[str, list[dict]]] = None, diff --git a/packages/agami-core/src/semantic_model/cli.py b/packages/agami-core/src/semantic_model/cli.py index 7e086cd7..833b1aa9 100644 --- a/packages/agami-core/src/semantic_model/cli.py +++ b/packages/agami-core/src/semantic_model/cli.py @@ -69,7 +69,7 @@ def cmd_validate(args) -> int: # exclusion as orphan_table_ref — even though curate passed and the runtime is fine. # A GENUINE orphan (a ref to a never-defined table) still fails, since no YAML # exists for it even with rejected included. - org = L.load_organization(args.root, include_rejected=True) + org = L.load_datasource(args.root, include_rejected=True) res = V.validate(org) print(V.format_result(res)) return 0 if res.ok else 1 @@ -97,19 +97,19 @@ def cmd_ensure_org_id(args) -> int: from . import build root = Path(args.root) - org_path = root / "datasource.yaml" - doc = L._read_yaml(org_path) or {} + datasource_path = root / "datasource.yaml" + doc = L._read_yaml(datasource_path) or {} existing = doc.get("org_id") oid = build.ensure_org_id(root, existing or None) if oid != existing: doc["org_id"] = oid - org_path.write_text(build._dump(doc), encoding="utf-8") + datasource_path.write_text(build._dump(doc), encoding="utf-8") print(oid) return 0 def cmd_context(args) -> int: - org = L.load_organization(args.root) + org = L.load_datasource(args.root) out = L.get_table_context( org, args.tables, @@ -122,13 +122,13 @@ def cmd_context(args) -> int: def cmd_bundle(args) -> int: - org = L.load_organization(args.root) + org = L.load_datasource(args.root) _print_json(L.get_subject_area_bundle(org, args.area)) return 0 def cmd_areas(args) -> int: - org = L.load_organization(args.root) + org = L.load_datasource(args.root) _print_json(RT.list_subject_areas(org)) return 0 @@ -138,8 +138,8 @@ def cmd_org_draft(args) -> int: # facts. Those are derived at read time (see `org-context`), so they never get baked into # the editable prose file where a human could clobber them. from . import org_draft - org = L.load_organization(args.root, include_rejected=False) - sys.stdout.write(org_draft.starter_organization_md(org)) + org = L.load_datasource(args.root, include_rejected=False) + sys.stdout.write(org_draft.starter_datasource_md(org)) return 0 @@ -151,7 +151,7 @@ def cmd_org_context(args) -> int: from . import org_draft from . import org_record as OR root = Path(args.root) - org = L.load_organization(root, include_rejected=False) + org = L.load_datasource(root, include_rejected=False) src_md = root / "datasource.md" source_narrative = src_md.read_text(encoding="utf-8") if src_md.exists() else "" art = root.parent # the artifacts dir holds the deployment-level record + company narrative @@ -194,7 +194,7 @@ def cmd_examples(args) -> int: def cmd_preflight(args) -> int: - org = L.load_organization(args.root) + org = L.load_datasource(args.root) result = RT.pre_flight_check(args.sql, org) _print_json(result.as_dict()) return 0 @@ -208,7 +208,7 @@ def cmd_prepare(args) -> int: sql = args.sql if args.sql_file: sql = Path(args.sql_file).read_text() - org = L.load_organization(args.root) + org = L.load_datasource(args.root) pf = RT.pre_flight_check(sql, org) if pf.risk and pf.action == "refuse": _print_json({"action": "refuse", "risk": pf.risk, "reason": pf.reason, @@ -240,7 +240,7 @@ def cmd_receipt(args) -> int: sql = args.sql if args.sql_file: sql = Path(args.sql_file).read_text() - org = L.load_organization(args.root) + org = L.load_datasource(args.root) pf = RT.pre_flight_check(sql, org) applied = json.loads(args.applied_filters) if args.applied_filters else None receipt = RT.assemble_receipt( @@ -256,14 +256,14 @@ def cmd_receipt(args) -> int: def cmd_review_queue(args) -> int: from . import curate - org = L.load_organization(args.root) + org = L.load_datasource(args.root) _print_json(curate.review_queue(org)) return 0 def cmd_review_items(args) -> int: from . import curate - org = L.load_organization(args.root, include_rejected=True) + org = L.load_datasource(args.root, include_rejected=True) _print_json(curate.all_items(org, scope=args.scope)) return 0 @@ -274,7 +274,7 @@ def cmd_curate_gate(args) -> int: skill running `sm sensitive` + `sm review-items --scope preseed` and branching by hand. Turn-boundary-safe (same answer on a fresh run or a resume).""" from . import curate - org = L.load_organization(args.root, include_rejected=True) + org = L.load_datasource(args.root, include_rejected=True) pii = curate.sensitive_columns(org).get("count", 0) preseed = len(curate.all_items(org, scope="preseed")) _print_json({"pii_count": pii, "preseed_count": preseed, @@ -284,7 +284,7 @@ def cmd_curate_gate(args) -> int: def cmd_model_tree(args) -> int: from . import curate - org = L.load_organization(args.root, include_rejected=True) + org = L.load_datasource(args.root, include_rejected=True) _print_json(curate.model_tree(org)) return 0 @@ -294,7 +294,7 @@ def cmd_coverage(args) -> int: The skill runs this at the end of Phase 2 (and reports it in the Phase 7 summary): `ok: false` with untouched columns means enrichment skipped the column pass.""" from . import curate - org = L.load_organization(args.root, include_rejected=True) + org = L.load_datasource(args.root, include_rejected=True) _print_json(curate.column_coverage(org)) return 0 @@ -304,7 +304,7 @@ def cmd_choice_coverage(args) -> int: runs this to confirm the value-enum decode ran — `ok: false` means coded columns are missing their {code:label} maps (the generator can't translate 'high' → 1 without them).""" from . import curate - org = L.load_organization(args.root, include_rejected=True) + org = L.load_datasource(args.root, include_rejected=True) _print_json(curate.unlabeled_choice_fields(org)) return 0 @@ -316,7 +316,7 @@ def cmd_sensitive(args) -> int: uses this count to decide whether to open the explorer (so the gate stops re-opening once the user has excluded the flagged columns).""" from . import curate - org = L.load_organization(args.root, include_rejected=True) + org = L.load_datasource(args.root, include_rejected=True) flagged = curate.sensitive_columns(org) suspected = curate.suspected_sensitive_columns(org) # `count`/`columns` keep their meaning (flagged PII — the gate signal); `suspected` is the @@ -359,7 +359,7 @@ def cmd_approve_queue(args) -> int: self-stamped `approve` op, and applies via curate.apply. `--kind` narrows to one item type; `--dry-run` prints the ops without touching the model.""" from . import curate - org = L.load_organization(args.root) + org = L.load_datasource(args.root) queue = curate.review_queue(org) kinds = set(args.kind) if args.kind else None ops = [{"op": "approve", "kind": it["kind"], "area": it["area"], @@ -415,7 +415,7 @@ def cmd_suggest_units(args) -> int: `discount`. Skips columns that already carry a `unit`. Emits {money_columns: [{area, table, column, type}]} for the caller to confirm + apply.""" from . import build as B - org = L.load_organization(args.root) + org = L.load_datasource(args.root) numeric = {"integer", "decimal", "float"} money = [] for sa in org.subject_areas: @@ -438,7 +438,7 @@ def cmd_set_units(args) -> int: if not unit: _print_json({"set": 0, "error": "pass --currency or --unit "}) return 1 - org = L.load_organization(args.root) + org = L.load_datasource(args.root) numeric = {"integer", "decimal", "float"} explicit = set(args.columns or []) ops = [] @@ -473,7 +473,7 @@ def cmd_suggest_metrics(args) -> int: from . import curate from . import dialects as D from . import introspect as INTRO - org = L.load_organization(args.root) + org = L.load_datasource(args.root) conn_type = {sc.name: sc.storage_type for sc in org.storage_connections} default_type = org.storage_connections[0].storage_type if org.storage_connections else "PostgreSQL" _dcache: dict = {} @@ -649,7 +649,7 @@ def cmd_seed_examples(args) -> int: the whole Phase-5 mechanical loop in one call (no throwaway validate-and-write script).""" from . import curate from .introspect import make_execute_sql_runner - org = L.load_organization(args.root, include_rejected=True) + org = L.load_datasource(args.root, include_rejected=True) # Gate 1 — enrichment completeness (NOT bypassable): every kept column must be described # or explicitly ai_unknown. Catches a model that enriched tables but skipped columns. block = _coverage_gate(org) @@ -699,7 +699,7 @@ def cmd_seed_validate(args) -> int: # answer — a column with unit: INR shows ₹ here too, not a bare number. If the model # can't load, fall back to raw cells (no regression). try: - fmt_org = L.load_organization(args.root) + fmt_org = L.load_datasource(args.root) except Exception: fmt_org = None items: list[dict] = [] @@ -804,10 +804,10 @@ def cmd_enrich_metadata(args) -> int: from . import dialects as D from . import introspect as INTRO from . import metadata_sources as MS - from .loader import load_organization + from .loader import load_datasource root = Path(args.root).expanduser() - org = load_organization(root) + org = load_datasource(root) model_tables = [t.name for sa in org.subject_areas for t in sa.tables_defined] valid = {(t.name.lower(), c.name.lower()) for sa in org.subject_areas for t in sa.tables_defined for c in t.columns} diff --git a/packages/agami-core/src/semantic_model/curate.py b/packages/agami-core/src/semantic_model/curate.py index de3a5a49..616abbe8 100644 --- a/packages/agami-core/src/semantic_model/curate.py +++ b/packages/agami-core/src/semantic_model/curate.py @@ -32,8 +32,8 @@ from . import validator as V from .loader import _read_yaml as _load -from .loader import load_organization -from .models import CrossSubjectAreaRelationship, Entity, Metric, Organization, Relationship +from .loader import load_datasource +from .models import CrossSubjectAreaRelationship, Datasource, Entity, Metric, Relationship # Process-lived incremental-validation cache (ACE-046). Every curation write re-validates the whole # model; keyed by (area content + table registry) so an enrichment run — which edits one area at a @@ -54,7 +54,7 @@ def _needs_review(obj) -> bool: return getattr(obj, "review_state", "approved") in ("unreviewed", "stale") -def review_queue(org: Organization) -> dict[str, Any]: +def review_queue(org: Datasource) -> dict[str, Any]: """Build the review queue. Rule 1 (metrics — block at runtime until signed off) is surfaced separately from Rule 2 (relationships / entities — lazy).""" rule1: list[dict] = [] @@ -99,7 +99,7 @@ def _tab(obj) -> str: return "manual" -def all_items(org: Organization, *, scope: str = "all") -> list[dict]: +def all_items(org: Datasource, *, scope: str = "all") -> list[dict]: """Every curatable entry (metric / relationship / entity), tab-classified, for the 4-tab review dashboard (For Review · Auto · Manual · Rejected). Tables and columns are curated in the model explorer, not here. @@ -161,7 +161,7 @@ def all_items(org: Organization, *, scope: str = "all") -> list[dict]: _MEANINGFUL_BLANK_TOLERANCE = 2 -def column_coverage(org: Organization) -> dict: +def column_coverage(org: Datasource) -> dict: """Per-table column-description coverage — the enrichment-completeness check. The signal is **table-level**, which keeps it conflict-free with the skill's @@ -249,7 +249,7 @@ def _self_evident(name: str) -> bool: } -def unlabeled_choice_fields(org: Organization) -> dict: +def unlabeled_choice_fields(org: Datasource) -> dict: """Coded columns whose `choice_field` skeleton still has BLANK labels — introspection seeded `{value: ""}` and the enrichment hasn't filled the meanings yet. The enrichment checks this (like `column_coverage`) to confirm the value-enum decode actually ran; @@ -273,7 +273,7 @@ def unlabeled_choice_fields(org: Organization) -> dict: "ok": not cols} -def sensitive_columns(org: Organization) -> dict: +def sensitive_columns(org: Datasource) -> dict: """Every column introspection (or a curator) flagged `sensitive` and hasn't excluded — the PII the agami-connect curate gate uses to decide whether to open the explorer. Deterministic + resumable (a turn-boundary-safe count, unlike remembering the @@ -289,7 +289,7 @@ def sensitive_columns(org: Organization) -> dict: return {"count": len(cols), "columns": cols} -def suspected_sensitive_columns(org: Organization) -> dict: +def suspected_sensitive_columns(org: Datasource) -> dict: """Columns the strict flag may have MISSED — `build.suspected_pii` matches the name but the column isn't marked `sensitive` (e.g. `first_name` in a non-PII-named table). Surfaced so a PII review catches false NEGATIVES, not just confirms hits. Excludes already-sensitive and @@ -358,7 +358,7 @@ def _entity_item(area: Optional[str], ent) -> dict: "maps_to": maps, **_trust(ent)} -def model_tree(org: Organization) -> dict[str, Any]: +def model_tree(org: Datasource) -> dict[str, Any]: """Browsable tree for the model explorer: area → table → columns, each with its review_state so the UI can show what's excluded. Load the org with include_rejected=True to see excluded entries here.""" @@ -423,12 +423,12 @@ def set_key_terminology(root: str | Path, terms: dict, *, merge: bool = True) -> Empty terms/definitions are dropped.""" root = Path(root) res = ApplyResult() - orgp = root / "datasource.yaml" - if not orgp.exists(): - res.errors.append(f"no datasource.yaml at {orgp}") + dsp = root / "datasource.yaml" + if not dsp.exists(): + res.errors.append(f"no datasource.yaml at {dsp}") return res - prior = orgp.read_text(encoding="utf-8") - odoc = _load(orgp) or {} + prior = dsp.read_text(encoding="utf-8") + odoc = _load(dsp) or {} existing = odoc.get("key_terminology") or {} if not isinstance(existing, dict): existing = {} @@ -439,17 +439,17 @@ def set_key_terminology(root: str | Path, terms: dict, *, merge: bool = True) -> odoc["key_terminology"] = merged else: odoc.pop("key_terminology", None) - _dump(orgp, odoc) + _dump(dsp, odoc) try: - vres = V.validate(load_organization(root, include_rejected=True), cache=_VALIDATION_CACHE) + vres = V.validate(load_datasource(root, include_rejected=True), cache=_VALIDATION_CACHE) res.validated = vres.ok if not vres.ok: res.errors = vres.errors - orgp.write_text(prior, encoding="utf-8") # revert, git-independent + dsp.write_text(prior, encoding="utf-8") # revert, git-independent return res except Exception as e: res.errors.append(f"validation failed to run: {e}") - orgp.write_text(prior, encoding="utf-8") + dsp.write_text(prior, encoding="utf-8") return res res.applied = [f"key_terminology: {len(merged)} term(s)"] res.committed = _git_commit(root, f"terminology: {len(merged)} term(s)") @@ -478,7 +478,7 @@ def apply(root: str | Path, ops: list[dict], *, signer: Optional[str] = None, # validate the whole model after the batch try: - org = load_organization(root, include_rejected=True) + org = load_datasource(root, include_rejected=True) vres = V.validate(org, cache=_VALIDATION_CACHE) res.validated = vres.ok if not vres.ok: @@ -632,16 +632,16 @@ def _apply_one(root: Path, op: dict, signer, role, return path # Cross-area (cross-schema / cross-datasource) join — it lives at the org level, not # in an area's relationships.yaml. Fall back to datasource.yaml's cross_subject_area_relationships. - orgp = root / "datasource.yaml" - _snapshot(backups, orgp) - odoc = _load(orgp) or {} + dsp = root / "datasource.yaml" + _snapshot(backups, dsp) + odoc = _load(dsp) or {} crels = odoc.get("cross_subject_area_relationships", []) chit = next((r for r in crels if r.get("from_table") == frm and r.get("to_table") == to), None) if chit is None: raise ValueError(f"relationship {name} not found in {path} or org cross-area relationships") _set_trust(chit, op, new_state, signer, role) - _dump(orgp, odoc) - return orgp + _dump(dsp, odoc) + return dsp raise ValueError(f"unknown kind {kind!r}") @@ -781,7 +781,7 @@ def write_items(root: str | Path, area: str, kind: str, items: list[dict], # validate the whole model; revert the batch on any failure try: - vres = V.validate(load_organization(root, include_rejected=True), cache=_VALIDATION_CACHE) + vres = V.validate(load_datasource(root, include_rejected=True), cache=_VALIDATION_CACHE) res.validated = vres.ok if not vres.ok: res.errors = vres.errors @@ -849,7 +849,7 @@ def add_relationships(root: str | Path, *, intra: Optional[dict[str, list[dict]] if not res.applied: return res try: - vres = V.validate(load_organization(root, include_rejected=True), cache=_VALIDATION_CACHE) + vres = V.validate(load_datasource(root, include_rejected=True), cache=_VALIDATION_CACHE) res.validated = vres.ok if not vres.ok: res.errors = vres.errors diff --git a/packages/agami-core/src/semantic_model/introspect.py b/packages/agami-core/src/semantic_model/introspect.py index 4127ce0c..354faa1a 100644 --- a/packages/agami-core/src/semantic_model/introspect.py +++ b/packages/agami-core/src/semantic_model/introspect.py @@ -58,7 +58,7 @@ from . import dialects as D from .models import ( Column, - Organization, + Datasource, PerformanceHints, Relationship, StorageConnection, @@ -198,7 +198,7 @@ def introspect( bigquery_region: str = "region-us", progress_path: Optional[str | Path] = None, append: bool = False, -) -> tuple[Organization, IntrospectReport]: +) -> tuple[Datasource, IntrospectReport]: """Introspect a live DB into the semantic model and (unless dry_run) write the canonical tree. `tables` (optional) is the allowlist of `schema.table` to build — the prune step's kept set (also the no-catalog case where enumeration is denied). @@ -279,8 +279,8 @@ def introspect( prev_rels: list[Relationship] = [] skip_keys: set = set() if append and not dry_run and (out / "datasource.yaml").exists(): - from .loader import load_organization - prev = load_organization(out, include_rejected=True) + from .loader import load_datasource + prev = load_datasource(out, include_rejected=True) batch_names = {t.name for t in built} for sa in prev.subject_areas: for t in sa.tables_defined: @@ -323,7 +323,7 @@ def introspect( storage_type=dialect.name, storage_config={"profile": profile, "credentials_ref": "/local/credentials"}, ) - org = Organization( + org = Datasource( datasource=profile, version=1, storage_connections=[storage], diff --git a/packages/agami-core/src/semantic_model/loader.py b/packages/agami-core/src/semantic_model/loader.py index 86b5ab5f..ec7c7bcc 100644 --- a/packages/agami-core/src/semantic_model/loader.py +++ b/packages/agami-core/src/semantic_model/loader.py @@ -17,7 +17,7 @@ cross_subject_area_metrics.yaml # optional, org-level prompt_examples//examples.yaml -The loader parses the tree into a single `Organization` model (so the validator +The loader parses the tree into a single `Datasource` model (so the validator and runtime work on one in-memory object). It also provides the context-assembly functions the runtime depends on: @@ -43,9 +43,9 @@ from .models import ( Column, CrossSubjectAreaRelationship, + Datasource, Entity, Metric, - Organization, Relationship, StorageConnection, SubjectArea, @@ -76,22 +76,22 @@ def _read_yaml(path: Path) -> Any: return yaml.safe_load(fh) -def load_organization(root: str | Path, *, include_rejected: bool = False) -> Organization: - """Parse a v2 profile directory into an Organization model. +def load_datasource(root: str | Path, *, include_rejected: bool = False) -> Datasource: + """Parse a v2 profile directory into an Datasource model. By default, entries the curator excluded (`review_state: rejected`) are dropped so the runtime never sees them. Pass `include_rejected=True` for the curation tools (agami-model), which must show excluded entries to toggle them. """ root = Path(root) - org_path = root / "datasource.yaml" - if not org_path.exists(): - raise FileNotFoundError(f"no datasource.yaml at {org_path}") - org_doc: dict[str, Any] = _read_yaml(org_path) or {} + datasource_path = root / "datasource.yaml" + if not datasource_path.exists(): + raise FileNotFoundError(f"no datasource.yaml at {datasource_path}") + ds_doc: dict[str, Any] = _read_yaml(datasource_path) or {} # storage connections — inline list OR refs into datasources//storage.yaml connections: list[StorageConnection] = [] - for sc in org_doc.get("storage_connections", []) or []: + for sc in ds_doc.get("storage_connections", []) or []: if isinstance(sc, dict) and "ref" in sc: ref_path = root / sc["ref"] connections.append(StorageConnection(**(_read_yaml(ref_path) or {}))) @@ -106,25 +106,25 @@ def load_organization(root: str | Path, *, include_rejected: bool = False) -> Or # subject areas — each referenced by directory name subject_areas: list[SubjectArea] = [] - for sa_ref in org_doc.get("subject_areas", []) or []: + for sa_ref in ds_doc.get("subject_areas", []) or []: sa_dir = root / (sa_ref if isinstance(sa_ref, str) else sa_ref.get("path", "")) if not sa_dir.exists(): # also accept a bare name under subject_areas/ sa_dir = root / "subject_areas" / str(sa_ref) subject_areas.append(_load_subject_area(sa_dir, include_rejected=include_rejected)) - org = Organization( - org_id=org_doc.get("org_id"), # F14: minted uuid4 (None for pre-F14 files) - datasource=org_doc.get("datasource", root.name), - version=org_doc.get("version", 1), - description=org_doc.get("description", ""), - fiscal_year_start_month=org_doc.get("fiscal_year_start_month", 1), + org = Datasource( + org_id=ds_doc.get("org_id"), # F14: minted uuid4 (None for pre-F14 files) + datasource=ds_doc.get("datasource", root.name), + version=ds_doc.get("version", 1), + description=ds_doc.get("description", ""), + fiscal_year_start_month=ds_doc.get("fiscal_year_start_month", 1), storage_connections=connections, subject_areas=subject_areas, - cross_subject_area_relationships=_load_cross_rels(root, org_doc), - cross_subject_area_entities=_load_cross_entities(root, org_doc), - cross_subject_area_metrics=_load_cross_metrics(root, org_doc), - key_terminology=org_doc.get("key_terminology", {}) or {}, + cross_subject_area_relationships=_load_cross_rels(root, ds_doc), + cross_subject_area_entities=_load_cross_entities(root, ds_doc), + cross_subject_area_metrics=_load_cross_metrics(root, ds_doc), + key_terminology=ds_doc.get("key_terminology", {}) or {}, ) return org @@ -133,14 +133,14 @@ def load_org_id(root: str | Path) -> str | None: """Return the minted org_id recorded in ``/datasource.yaml``, or ``None`` if the file is absent or predates F14 (no ``org_id`` key). This is the serve-time identity read (F14 / ACE-056). - Deliberately read-only and lenient — unlike ``load_organization`` it never raises on a missing + Deliberately read-only and lenient — unlike ``load_datasource`` it never raises on a missing file — so the single-tenant resolver can fall through to its ``"local"`` default. Reads only the top-level key; it does NOT build the full pydantic model (identity resolution runs per process, not per query, so it stays cheap).""" - org_path = Path(root) / "datasource.yaml" - if not org_path.exists(): + datasource_path = Path(root) / "datasource.yaml" + if not datasource_path.exists(): return None - doc = _read_yaml(org_path) or {} + doc = _read_yaml(datasource_path) or {} oid = doc.get("org_id") return oid or None @@ -277,10 +277,10 @@ def _load_subject_area(sa_dir: Path, include_rejected: bool = False) -> SubjectA ) -def _load_cross_rels(root: Path, org_doc: dict) -> list[CrossSubjectAreaRelationship]: +def _load_cross_rels(root: Path, ds_doc: dict) -> list[CrossSubjectAreaRelationship]: out: list[CrossSubjectAreaRelationship] = [] # inline on datasource.yaml - for r in org_doc.get("cross_subject_area_relationships", []) or []: + for r in ds_doc.get("cross_subject_area_relationships", []) or []: out.append(CrossSubjectAreaRelationship(**r)) # or a sidecar file f = root / "cross_subject_area_relationships.yaml" @@ -291,8 +291,8 @@ def _load_cross_rels(root: Path, org_doc: dict) -> list[CrossSubjectAreaRelation return out -def _load_cross_entities(root: Path, org_doc: dict) -> list[Entity]: - out = [Entity(**e) for e in (org_doc.get("cross_subject_area_entities", []) or [])] +def _load_cross_entities(root: Path, ds_doc: dict) -> list[Entity]: + out = [Entity(**e) for e in (ds_doc.get("cross_subject_area_entities", []) or [])] f = root / "cross_subject_area_entities.yaml" if f.exists(): doc = _read_yaml(f) or {} @@ -301,8 +301,8 @@ def _load_cross_entities(root: Path, org_doc: dict) -> list[Entity]: return out -def _load_cross_metrics(root: Path, org_doc: dict) -> list[Metric]: - out = [Metric(**mm) for mm in (org_doc.get("cross_subject_area_metrics", []) or [])] +def _load_cross_metrics(root: Path, ds_doc: dict) -> list[Metric]: + out = [Metric(**mm) for mm in (ds_doc.get("cross_subject_area_metrics", []) or [])] f = root / "cross_subject_area_metrics.yaml" if f.exists(): doc = _read_yaml(f) or {} @@ -351,7 +351,7 @@ def find(self, table_name: str, area: Optional[str] = None) -> Optional[Table]: return self._pick(self.org_wide, table_name, bare) -def build_table_index(org: Organization) -> TableIndex: +def build_table_index(org: Datasource) -> TableIndex: """Build the O(1) name→Table index once per schema call (ACE-047). Scan order (areas, then tables) matches `_find_table`, and first-occurrence-wins via setdefault keeps a clash resolving identically.""" @@ -374,7 +374,7 @@ def build_table_index(org: Organization) -> TableIndex: def _find_table( - org: Organization, table_name: str, area: Optional[str] = None, + org: Datasource, table_name: str, area: Optional[str] = None, *, index: Optional[TableIndex] = None, ) -> Optional[Table]: if index is not None: @@ -402,7 +402,7 @@ def _table_alias(table_name: str) -> str: def collect_default_filters( - org: Organization, + org: Datasource, table_names: Iterable[str], *, area: Optional[str] = None, @@ -474,7 +474,7 @@ def _exposed_groups_for(sa: SubjectArea, table_name: str) -> Optional[list[str]] def get_table_context( - org: Organization, + org: Datasource, tables: list[str], *, area: Optional[str] = None, @@ -560,7 +560,7 @@ def _column_detail(col: Column, include: list[str]) -> dict[str, Any]: def _relationships_among( - org: Organization, tables: list[str], area: Optional[str] + org: Datasource, tables: list[str], area: Optional[str] ) -> list[dict[str, Any]]: names = {_table_alias(t) for t in tables} | set(tables) out: list[dict[str, Any]] = [] @@ -578,7 +578,7 @@ def _relationships_among( return out -def _metrics_for(org: Organization, tables: list[str], area: Optional[str]) -> list[dict[str, Any]]: +def _metrics_for(org: Datasource, tables: list[str], area: Optional[str]) -> list[dict[str, Any]]: names = {_table_alias(t) for t in tables} | set(tables) out: list[dict[str, Any]] = [] areas = [org.subject_area(area)] if area else org.subject_areas @@ -591,7 +591,7 @@ def _metrics_for(org: Organization, tables: list[str], area: Optional[str]) -> l return out -def get_subject_area_bundle(org: Organization, area: str) -> dict[str, Any]: +def get_subject_area_bundle(org: Datasource, area: str) -> dict[str, Any]: """One-shot bundle for small subject areas (a few dozen tables).""" sa = org.subject_area(area) if sa is None: @@ -624,7 +624,7 @@ def list_prompt_examples(root: str | Path, area: str, """Load scope-tagged examples for a subject area (prompt_examples//examples.yaml). Examples the curator rejected (`status: rejected`) are dropped by default so the - runtime ranker never anchors on them — mirroring how `load_organization` drops + runtime ranker never anchors on them — mirroring how `load_datasource` drops `review_state: rejected` model entries. Pass `include_rejected=True` for the curation view (re-render, dedup, audit), where a rejected example must still be visible. """ @@ -639,7 +639,7 @@ def list_prompt_examples(root: str | Path, area: str, __all__ = [ - "load_organization", + "load_datasource", "load_org_id", "deployment_org_id", "collect_default_filters", diff --git a/packages/agami-core/src/semantic_model/models.py b/packages/agami-core/src/semantic_model/models.py index 55844045..5432b4c6 100644 --- a/packages/agami-core/src/semantic_model/models.py +++ b/packages/agami-core/src/semantic_model/models.py @@ -2,7 +2,7 @@ Hierarchy (see the design doc's "The new hierarchy" section): - Organization + Datasource ├─ description ├─ storage_connections[] (physical — host, port, creds, dialect) ├─ subject_areas[] (logical — the primary semantic unit) @@ -627,11 +627,11 @@ def defined_table(self, name: str) -> Optional[Table]: # --------------------------------------------------------------------------- -# Organization (top level) +# Datasource (top level) # --------------------------------------------------------------------------- -class Organization(_Base): +class Datasource(_Base): # Locally-minted, globally-unique deployment identity (F14 / ACE-056): a uuid4 hex minted ONCE at # agami-connect and persisted in datasource.yaml, then immutable. Optional/None so pre-F14 datasource.yaml files # (which have no org_id key) still load under the model's `extra="forbid"` policy. Never transmitted; @@ -677,8 +677,8 @@ def storage_connection(self, name: str) -> Optional[StorageConnection]: # --------------------------------------------------------------------------- -# Organization record (F15 / ACE-067) — the deployment-level company record that -# sits ABOVE the per-datasource Organization models. One per artifacts dir. +# Company record (F15 / ACE-067) — the deployment-level company record (OrgRecord) that +# sits ABOVE the per-datasource Datasource models. One per artifacts dir. # --------------------------------------------------------------------------- @@ -707,7 +707,7 @@ class OrgRecord(_Base): description: Optional[str] = None fiscal_year_start_month: Optional[int] = None display_conventions: DisplayConventions = Field(default_factory=DisplayConventions) - # Company-wide glossary: term -> one-line definition. Same shape as Organization.key_terminology + # Company-wide glossary: term -> one-line definition. Same shape as Datasource.key_terminology # (the established glossary type), but scoped to the COMPANY rather than a single datasource. glossary: dict[str, str] = Field(default_factory=dict) # The datasources (profile names) attached under this org. Auto-maintained: rebuilt from the profile @@ -717,7 +717,7 @@ class OrgRecord(_Base): @field_validator("fiscal_year_start_month") @classmethod def _fy_month(cls, v: Optional[int]) -> Optional[int]: - # Same 1..12 bound as Organization, but None is allowed (an unauthored record has no fiscal year). + # Same 1..12 bound as Datasource, but None is allowed (an unauthored record has no fiscal year). if v is not None and not 1 <= v <= 12: raise ValueError("fiscal_year_start_month must be 1..12") return v @@ -754,7 +754,7 @@ def _fy_month(cls, v: Optional[int]) -> Optional[int]: "Relationship", "CrossSubjectAreaRelationship", "SubjectArea", - "Organization", + "Datasource", "DisplayConventions", "OrgRecord", # constants diff --git a/packages/agami-core/src/semantic_model/org_draft.py b/packages/agami-core/src/semantic_model/org_draft.py index 69b746d7..5f32d058 100644 --- a/packages/agami-core/src/semantic_model/org_draft.py +++ b/packages/agami-core/src/semantic_model/org_draft.py @@ -4,7 +4,7 @@ * **datasource.md** is the human's narrative ONLY (what the company/product is, who the users are). agami never writes facts into it, so there is nothing for a human to - accidentally overwrite or delete. `starter_organization_md()` is the blank-path prompt. + accidentally overwrite or delete. `starter_datasource_md()` is the blank-path prompt. * **The factual context** — shape, subject areas, conventions, and the decoded glossary — is `derived_context()`, computed FRESH from the structured model at read time. The glossary lives in the structured `key_terminology` field, not inline prose, so it always reaches the @@ -21,7 +21,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from .models import Organization, OrgRecord + from .models import Datasource, OrgRecord def _plural(n: int, word: str) -> str: @@ -47,7 +47,7 @@ def _strip_comments(text: str) -> str: return re.sub(r"\n{3,}", "\n\n", out).strip() -def derived_context(org: "Organization", *, with_curated_glossary: bool = True) -> str: +def derived_context(org: "Datasource", *, with_curated_glossary: bool = True) -> str: """The model-DERIVED factual context: shape + subject areas + conventions + glossary. Computed fresh from the structured model every time and NOT persisted into datasource.md @@ -102,12 +102,12 @@ def derived_context(org: "Organization", *, with_curated_glossary: bool = True) return "\n".join(lines).strip() -def compose_context(human_md: str, org: "Organization") -> str: +def compose_context(human_md: str, org: "Datasource") -> str: """Read-time assembly of the full org context: the human's narrative (HTML comments stripped) followed by the model-derived summary under its OWN heading. The two parts stay SEPARATE — the human's prose is never mixed with auto content, so nothing can be accidentally overwritten. Either part may be empty. Used by the MCP, the query skill, and - the explorer's Organization view.""" + the explorer's Datasource view.""" human = _strip_comments(human_md) derived = derived_context(org) parts: list[str] = [] @@ -153,7 +153,7 @@ def _company_block(record: "OrgRecord", narrative: str) -> str: return "\n".join(lines).strip() -def _source_block(org: "Organization", narrative: str) -> str: +def _source_block(org: "Datasource", narrative: str) -> str: """One datasource's context: its optional source-specific narrative + the model-derived summary, under a heading naming the datasource (so a federated answer keeps the vocabularies apart).""" seg = [f"## {org.datasource} — datasource context"] @@ -168,7 +168,7 @@ def _source_block(org: "Organization", narrative: str) -> str: def compose_org_context( org_record: "OrgRecord | None", - ontologies: "list[Organization]", + ontologies: "list[Datasource]", *, company_narrative: str = "", source_narratives: "list[str] | None" = None, @@ -199,7 +199,7 @@ def compose_org_context( return "\n\n".join(p for p in parts if p).strip() -def starter_organization_md(org: "Organization") -> str: +def starter_datasource_md(org: "Datasource") -> str: """A human-narrative STARTER for the skip path — never blank. It seeds `# About this database` with a one-line factual SUMMARY drawn from the model (org + what the subject areas cover) so the section reads as something, then invites the human to make it theirs. @@ -227,7 +227,7 @@ def starter_organization_md(org: "Organization") -> str: ]) -def draft_organization_md(org: "Organization") -> str: +def draft_datasource_md(org: "Datasource") -> str: """Back-compat: the model-derived context as a standalone document (no human prose). Prefer compose_context()/derived_context() in new code.""" return compose_context("", org) @@ -240,7 +240,7 @@ def draft_organization_md(org: "Organization") -> str: _MAX_ENUM_COLS = 25 -def _key_terminology(lines: list[str], org: "Organization", areas: list, +def _key_terminology(lines: list[str], org: "Datasource", areas: list, include_curated: bool = True) -> None: """Append the glossary: the curated `key_terminology` terms (only when `include_curated`) plus auto-derived enum legends from `choice_field` columns. Omitted entirely when there's diff --git a/packages/agami-core/src/semantic_model/runtime.py b/packages/agami-core/src/semantic_model/runtime.py index a9cc0733..3bdc0509 100644 --- a/packages/agami-core/src/semantic_model/runtime.py +++ b/packages/agami-core/src/semantic_model/runtime.py @@ -1,7 +1,7 @@ """Runtime traversal for the agami semantic-model-v2 path. Implements the design doc's "Traversal" + "Runtime walkthrough" primitives as -pure functions over a parsed `Organization` model, so they're equally usable from +pure functions over a parsed `Datasource` model, so they're equally usable from the MCP server (`mcp_server.py`) and the skill CLI, and fully unit-testable without a live database. Anything that needs to touch the DB (entity probing) is injected as a `probe` callable — the caller wires in a real prober; tests pass a @@ -45,9 +45,9 @@ from .models import ( Column, + Datasource, Entity, Metric, - Organization, Relationship, ) from .models import ( @@ -93,7 +93,7 @@ def _parse_sql(sql: str) -> "exp.Expression | None": return None -def build_guard_context(sql: str, org: Organization) -> "GuardContext | None": +def build_guard_context(sql: str, org: Datasource) -> "GuardContext | None": """Parse `sql` once and build each guard index once, so the _model_safety battery shares them instead of every guard redoing the work (audit P2 / ACE-045). Returns None when sqlglot is unavailable: every guard then short-circuits to allow before it touches the context, so @@ -122,7 +122,7 @@ def build_guard_context(sql: str, org: Organization) -> "GuardContext | None": # --------------------------------------------------------------------------- -def list_subject_areas(org: Organization) -> list[dict[str, Any]]: +def list_subject_areas(org: Datasource) -> list[dict[str, Any]]: """Compact listing for area selection — also the one-call model map. The counts tell a caller the whole shape of each area (and where things live: relationships and entities/metrics are area-level, not per-table) without reading any YAML.""" @@ -180,7 +180,7 @@ def is_high_confidence(matches: list[ExampleMatch]) -> bool: # --------------------------------------------------------------------------- -def _area_entities(org: Organization, area: Optional[str]) -> list[tuple[Optional[str], Entity]]: +def _area_entities(org: Datasource, area: Optional[str]) -> list[tuple[Optional[str], Entity]]: out: list[tuple[Optional[str], Entity]] = [] for sa in org.subject_areas: if area and sa.name != area: @@ -193,7 +193,7 @@ def _area_entities(org: Organization, area: Optional[str]) -> list[tuple[Optiona def resolve_entities( - query: str, org: Organization, *, area: Optional[str] = None, top_k: int = 5 + query: str, org: Datasource, *, area: Optional[str] = None, top_k: int = 5 ) -> list[dict[str, Any]]: """Lexically match query terms to entity name / plural / other_names.""" q = query.lower() @@ -226,7 +226,7 @@ def resolve_entities( def resolve_metrics( - query: str, org: Organization, *, area: Optional[str] = None, top_k: int = 5 + query: str, org: Datasource, *, area: Optional[str] = None, top_k: int = 5 ) -> list[dict[str, Any]]: from . import derived as _D @@ -285,7 +285,7 @@ class IdentifyResult: def identify_entity( literal: str, - org: Organization, + org: Datasource, *, area: Optional[str] = None, probe: Optional[Prober] = None, @@ -437,7 +437,7 @@ def as_dict(self) -> dict[str, Any]: "reason": self.reason, "suggestion": self.suggestion} -def _sensitive_by_table(org: Organization) -> tuple[dict[str, set[str]], set[str]]: +def _sensitive_by_table(org: Datasource) -> tuple[dict[str, set[str]], set[str]]: """(table name -> {sensitive column names}, union of all sensitive column names).""" by_table: dict[str, set[str]] = {} allnames: set[str] = set() @@ -498,7 +498,7 @@ def _output_selects(node: "exp.Expression") -> list["exp.Select"]: return [] -def check_sensitive_projection(sql: str, org: Organization, +def check_sensitive_projection(sql: str, org: Datasource, ctx: "GuardContext | None" = None) -> SensitiveCheckResult: """Refuse a query that PROJECTS a `sensitive` column's raw values; allow the column in COUNT, filters, GROUP BY, and joins. Degrades to allow when sqlglot @@ -586,7 +586,7 @@ def as_dict(self) -> dict[str, Any]: "reason": self.reason, "suggestion": self.suggestion} -def check_table_scope(sql: str, org: Organization, +def check_table_scope(sql: str, org: Datasource, ctx: "GuardContext | None" = None) -> TableScopeResult: """Refuse a query that references a table not declared in the semantic model. @@ -713,7 +713,7 @@ def as_dict(self) -> dict[str, Any]: "reason": self.reason, "suggestion": self.suggestion} -def check_column_scope(sql: str, org: Organization, +def check_column_scope(sql: str, org: Datasource, ctx: "GuardContext | None" = None) -> ColumnScopeResult: """Refuse a query that references a column not declared on the table it binds to. @@ -850,7 +850,7 @@ def _select_chain(node): ) -def _cardinality_index(org: Organization) -> list[Relationship]: +def _cardinality_index(org: Datasource) -> list[Relationship]: rels: list[Relationship] = [] for sa in org.subject_areas: rels.extend(sa.relationships) @@ -881,7 +881,7 @@ def _many_side_facing_one(rels: list[Relationship], table: str, dim: str) -> boo return False -def pre_flight_check(sql: str, org: Organization, +def pre_flight_check(sql: str, org: Datasource, ctx: "GuardContext | None" = None) -> PreFlightResult: """Detect fan-trap / chasm-trap and decide rewrite-vs-refuse-vs-allow. @@ -910,7 +910,7 @@ def pre_flight_check(sql: str, org: Organization, return PreFlightResult(None, "allow", sql, reason="no fan/chasm or aggregation issue in any arm") -def _preflight_select(tree: "exp.Select", org: Organization, sql: str, allow_rewrite: bool, +def _preflight_select(tree: "exp.Select", org: Datasource, sql: str, allow_rewrite: bool, ctx: "GuardContext | None" = None) -> PreFlightResult: """Fan/chasm + aggregation-semantics analysis of a SINGLE SELECT. `sql` is that select's own text (used for the join rewrite + messages). When `allow_rewrite` is @@ -1020,7 +1020,7 @@ def _preflight_select(tree: "exp.Select", org: Organization, sql: str, allow_rew # --------------------------------------------------------------------------- -def _column_index(org: Organization) -> dict[str, dict[str, Column]]: +def _column_index(org: Datasource) -> dict[str, dict[str, Column]]: """bare table name -> {column name -> Column}.""" idx: dict[str, dict[str, Column]] = {} for sa in org.subject_areas: @@ -1053,7 +1053,7 @@ def _bare_aggregate_column(agg: "exp.AggFunc") -> Optional["exp.Column"]: return None -def _semi_additive_columns(org: Organization) -> dict[tuple[str, str], "Metric"]: +def _semi_additive_columns(org: Datasource) -> dict[tuple[str, str], "Metric"]: """(table, column) -> the semi-additive Metric that SUMs it (declares non_additive_dimensions). Keyed by (table, column) — NOT bare column name — so two tables that both have a `balance` don't cross-contaminate. The table is the binding's @@ -1102,7 +1102,7 @@ def _groups_by_time(tree: "exp.Select", scope: dict[str, str], def _check_aggregation_semantics( - tree: "exp.Select", org: Organization, scope: dict[str, str], sql: str, + tree: "exp.Select", org: Datasource, scope: dict[str, str], sql: str, ctx: "GuardContext | None" = None, ) -> Optional[PreFlightResult]: colidx = ctx.column_index if ctx is not None else _column_index(org) @@ -1220,7 +1220,7 @@ def build_receipt( return receipt -def _model_table_index(org: Organization) -> dict[str, tuple]: +def _model_table_index(org: Datasource) -> dict[str, tuple]: """bare table name -> (Table, area_name). First occurrence wins (a cross-schema name clash is rare and the relationships now carry schema to disambiguate).""" idx: dict[str, tuple] = {} @@ -1235,7 +1235,7 @@ def _norm_sql(s: Optional[str]) -> str: def assemble_receipt( - org: Organization, + org: Datasource, sql: str, *, model_version: Optional[str] = None, @@ -1499,7 +1499,7 @@ def _drop_fanout_joins(sql: str, drop_tables: set[str]) -> Optional[str]: def apply_default_filters( sql: str, - org: Organization, + org: Datasource, *, area: Optional[str] = None, params: Optional[dict[str, str]] = None, @@ -1594,7 +1594,7 @@ def _term_score(query_lower: str, name: str) -> float: return 0.0 -def resolve_result_units(org: Organization, sql: str) -> dict[str, str]: +def resolve_result_units(org: Datasource, sql: str) -> dict[str, str]: """Map each SELECT output column -> display unit, **tracing the SQL** (not matching names): an aggregate/expression over a column inherits that column's unit, so `SUM(amount) AS total_outstanding` correctly resolves to amount's currency — the diff --git a/packages/agami-core/src/semantic_model/validator.py b/packages/agami-core/src/semantic_model/validator.py index 162f328e..8b45cd30 100644 --- a/packages/agami-core/src/semantic_model/validator.py +++ b/packages/agami-core/src/semantic_model/validator.py @@ -51,8 +51,8 @@ from .models import ( CrossSubjectAreaRelationship, + Datasource, Entity, - Organization, Relationship, SubjectArea, Table, @@ -147,7 +147,7 @@ def _org_tables_sig(org_tables: dict[str, Table]) -> str: return h.hexdigest() -def _bound_cache(cache: ValidationCache, org: Organization) -> None: +def _bound_cache(cache: ValidationCache, org: Datasource) -> None: """Cap total cache size for long-lived processes. Evict oldest entries for areas NOT in the current model first (insertion order); never evict the current model's areas, so a single large model doesn't thrash. Only triggers well past any real model's area count.""" @@ -161,7 +161,7 @@ def _bound_cache(cache: ValidationCache, org: Organization) -> None: def _validate_area( - sa: SubjectArea, org: Organization, org_tables: dict[str, Table] + sa: SubjectArea, org: Datasource, org_tables: dict[str, Table] ) -> list[Finding]: """Run one area's per-area rule battery into its own result and return just its findings, so a run cache can reuse them verbatim when the area (and the table registry) are unchanged.""" @@ -183,7 +183,7 @@ def _validate_area( return ares.findings -def validate(org: Organization, *, cache: "ValidationCache | None" = None) -> ValidationResult: +def validate(org: Datasource, *, cache: "ValidationCache | None" = None) -> ValidationResult: """Run every cross-cutting rule. Returns a ValidationResult (never raises on a model-level problem — those surface as the model failing to parse upstream). @@ -237,7 +237,7 @@ def _all_tables(sa: SubjectArea) -> dict[str, Table]: return {t.name: t for t in sa.tables_defined} -def _check_storage_connection_refs(org: Organization, res: ValidationResult) -> None: +def _check_storage_connection_refs(org: Datasource, res: ValidationResult) -> None: known = {sc.name for sc in org.storage_connections} for sa in org.subject_areas: for ref in sa.tables: @@ -268,7 +268,7 @@ def _check_subject_area_sizing(sa: SubjectArea, res: ValidationResult) -> None: ) -def _org_tables(org: Organization) -> dict[str, Table]: +def _org_tables(org: Datasource) -> dict[str, Table]: """Every canonical Table across the org, keyed by name (org-wide resolution).""" out: dict[str, Table] = {} for sa in org.subject_areas: @@ -420,7 +420,7 @@ def _check_entity_mappings( def _check_relationship( rel: Relationship, sa: SubjectArea, - org: Organization, + org: Datasource, res: ValidationResult, *, cross: bool, @@ -465,7 +465,7 @@ def _check_relationship( def _check_cross_relationship( - rel: CrossSubjectAreaRelationship, org: Organization, res: ValidationResult + rel: CrossSubjectAreaRelationship, org: Datasource, res: ValidationResult ) -> None: sa_from = org.subject_area(rel.from_subject_area) sa_to = org.subject_area(rel.to_subject_area) @@ -508,7 +508,7 @@ def _check_cross_relationship( ) -def _check_cross_area_entity_collisions(org: Organization, res: ValidationResult) -> None: +def _check_cross_area_entity_collisions(org: Datasource, res: ValidationResult) -> None: """Two areas independently declaring the same entity name (or overlapping other_names) with maps_to into different connections/schemas → warning + a suggestion to unify in a cross-cutting area.""" @@ -557,7 +557,7 @@ def _check_cross_area_entity_collisions(org: Organization, res: ValidationResult ) -def _check_metric_backend_neutrality(org: Organization, res: ValidationResult) -> None: +def _check_metric_backend_neutrality(org: Datasource, res: ValidationResult) -> None: """Verification check #4: every metric's `calculation` (prose intent) must be non-empty — no metric may depend ONLY on a SQL binding. (models enforces non-empty; here we also flag binding-only metrics with placeholder prose.)""" @@ -573,7 +573,7 @@ def _check_metric_backend_neutrality(org: Organization, res: ValidationResult) - ) -def _check_derived_metrics(org: Organization, res: ValidationResult) -> None: +def _check_derived_metrics(org: Datasource, res: ValidationResult) -> None: """Scorecard #1: a derived metric (one composing others via {base} placeholders) must resolve — no cycles, no unknown bases, no illegal second-order nesting. A base over a disjoint grain is a warning (inline composition may be wrong; full @@ -630,7 +630,7 @@ def _binding_column_refs(sql: str) -> set[str]: return {c.name.lower() for c in tree.find_all(exp.Column) if c.name} -def _check_metric_binding_columns(org: Organization, res: ValidationResult) -> None: +def _check_metric_binding_columns(org: Datasource, res: ValidationResult) -> None: """WARN when a metric's binding references a column that exists on NONE of its source_tables — catches a typo'd or renamed column (e.g. `SUM(cst)`) before it fails at query time, which is the one thing the hand-edited SQL snippet otherwise has no safety net for. @@ -735,7 +735,7 @@ def _types_compatible(a: str, b: str) -> bool: return frozenset({a, b}) in _COERCIBLE_PAIRS -def _entity_targets(org: Organization, area_name: str, ent: Entity) -> set[str]: +def _entity_targets(org: Datasource, area_name: str, ent: Entity) -> set[str]: """Resolve an entity's maps_to into a set of 'connection.schema.table' strings.""" sa = org.subject_area(area_name) out: set[str] = set() diff --git a/packages/agami-core/src/tools.py b/packages/agami-core/src/tools.py index 75e772cd..b7eb5bde 100644 --- a/packages/agami-core/src/tools.py +++ b/packages/agami-core/src/tools.py @@ -205,7 +205,7 @@ def check_read_only(sql: str) -> str | None: def _load_org(profile: str): - """Lazily load the semantic model for a profile, producing an `Organization`. Two backends + """Lazily load the semantic model for a profile, producing an `Datasource`. Two backends behind one seam: when AGAMI_DB_URL is set the hosted server reads it from the DB; otherwise the local skill reads the YAML files (unchanged). Raises a clear error if the model deps (pydantic) aren't importable or there's no model for the profile.""" @@ -213,7 +213,7 @@ def _load_org(profile: str): store = Store.from_env() if store is not None: - from model_store import load_organization as _load_db + from model_store import load_datasource as _load_db try: org = _load_db(store, profile, org_id=_current_org_id()) @@ -234,7 +234,7 @@ def _load_org(profile: str): f"No semantic model at {root}/datasource.yaml. Run the agami-connect skill to " f"introspect this database." ) - return L.load_organization(root) + return L.load_datasource(root) def _resolve_units(profile: str, sql: str) -> dict[str, str]: @@ -358,7 +358,7 @@ def _credential_org_id() -> str: def get_cached_org(profile: str): """Load the semantic model for `profile`, cached per process and keyed (org, datasource, version). - Reuses one Organization across the loads within a query AND across queries, until the model version + Reuses one Datasource across the loads within a query AND across queries, until the model version changes; a cache miss falls back to a fresh `_load_org`.""" version = _model_version(profile) # cheap: one DB row / dir listing, not a full model load if version is None: @@ -387,7 +387,7 @@ def _context_sources(profile: str, org_id: str) -> "tuple[str, str | None, Any, datasource.md, USER_MEMORY.md, the deployment ``OrgRecord``, and the company narrative. Under the DB backend all of it is read on a SINGLE connection — this is a hot tool path, so open ``Store`` once, not per-source; with no DB configured it falls back to file reads (a DB deploy reads no files at runtime). - Returns ``(org_md, user_md, record | None, company_md)``; missing pieces come back empty/``None`` so the + Returns ``(datasource_md, user_md, record | None, company_md)``; missing pieces come back empty/``None`` so the two-level composition degrades cleanly.""" from store import Store diff --git a/plugins/agami/lib/execute_sql.py b/plugins/agami/lib/execute_sql.py index fb0cd20a..865e3a9c 100644 --- a/plugins/agami/lib/execute_sql.py +++ b/plugins/agami/lib/execute_sql.py @@ -867,7 +867,7 @@ def _hosted() -> bool: def _resolve_guard_model(profile: str): """Resolve the semantic model for the safety pass, mirroring `tools._load_org` (ACE-051): from the DB when one is configured (hosted — the `/artifacts` disk mount may be absent), else the - on-disk YAML (local). Returns an `Organization` or None if neither is available. + on-disk YAML (local). Returns an `Datasource` or None if neither is available. The DB import is lazy AND env-guarded on purpose: the local executor runs from a stdlib-lean mirror that does not ship `store`/`model_store`, so we only reach for them when a DB is set. @@ -881,7 +881,7 @@ def _resolve_guard_model(profile: str): # itself, not a diagnostic line. if _hosted(): try: - from model_store import load_organization as _load_db + from model_store import load_datasource as _load_db from store import Store store = Store.from_env() @@ -898,7 +898,7 @@ def _resolve_guard_model(profile: str): root = Path(os.environ.get("AGAMI_ARTIFACTS_DIR") or (Path.home() / "agami-artifacts")) / profile if (root / "datasource.yaml").exists(): try: - return L.load_organization(root) + return L.load_datasource(root) except Exception: pass # unparseable/absent on disk -> None (hosted then fails closed) return None diff --git a/plugins/agami/lib/semantic_model/__init__.py b/plugins/agami/lib/semantic_model/__init__.py index 55a107fd..f3524a4a 100644 --- a/plugins/agami/lib/semantic_model/__init__.py +++ b/plugins/agami/lib/semantic_model/__init__.py @@ -1,6 +1,6 @@ """agami semantic model — provider-portable, standard-concepts hierarchy. -This is **the** agami semantic model: a hierarchy of Organization → Storage +This is **the** agami semantic model: a hierarchy of Datasource → Storage Connection (physical) + Subject Area (logical) → Table / Entity / Metric / Relationship, with provider-portable declarative fields (default_filters, value_transform, caveats, value_pattern, sensitive, cardinality, …) so that any diff --git a/plugins/agami/scripts/parse_model_feedback.py b/plugins/agami/scripts/parse_model_feedback.py index 1835e139..3ec161d6 100644 --- a/plugins/agami/scripts/parse_model_feedback.py +++ b/plugins/agami/scripts/parse_model_feedback.py @@ -48,7 +48,7 @@ _LIST_KEYS = {"exclude tables", "include tables", "exclude columns", "include columns"} _JSON_KEYS = {"curate-ops", "new-metrics", "key-terminology", "example-edits", "new-examples"} -_KEYS = _LIST_KEYS | _JSON_KEYS | {"profile", "signed-off-by", "organization-md"} +_KEYS = _LIST_KEYS | _JSON_KEYS | {"profile", "signed-off-by", "datasource-md"} _TABLE_QNAME = re.compile(r"^[A-Za-z0-9_]+\.[A-Za-z0-9_]+$") _COLUMN_QNAME = re.compile(r"^[A-Za-z0-9_]+\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+$") @@ -89,7 +89,7 @@ def parse(text: str) -> tuple[dict, list, dict | None]: needs: dict | None = None ops: list = [] data: dict = {"profile": None, "ops": ops, "new_metrics_by_area": {}, - "examples_by_area": {}, "key_terminology": None, "organization_md": None, + "examples_by_area": {}, "key_terminology": None, "datasource_md": None, "signer": None, "role": None} if "profile" in sec: @@ -124,7 +124,7 @@ def _qnames(val: str) -> list[str]: entry["column"] = parts[2] ops.append(entry) - for jk in ("curate-ops", "new-metrics", "key-terminology", "example-edits", "new-examples", "organization-md"): + for jk in ("curate-ops", "new-metrics", "key-terminology", "example-edits", "new-examples", "datasource-md"): if jk not in sec: continue try: @@ -155,8 +155,8 @@ def _qnames(val: str) -> list[str]: data["examples_by_area"].setdefault(area, []).append(ex) elif jk == "key-terminology": data["key_terminology"] = parsed - elif jk == "organization-md": - data["organization_md"] = parsed # JSON-encoded string → the full datasource.md text + elif jk == "datasource-md": + data["datasource_md"] = parsed # JSON-encoded string → the full datasource.md text if bad_targets: needs = {"kind": "malformed_targets", "targets": bad_targets, diff --git a/plugins/agami/scripts/render_model_explorer.py b/plugins/agami/scripts/render_model_explorer.py index a6fbe474..e2dcf5ec 100644 --- a/plugins/agami/scripts/render_model_explorer.py +++ b/plugins/agami/scripts/render_model_explorer.py @@ -53,7 +53,7 @@ def build_manifest(profile_dir: Path, profile: str) -> dict: from semantic_model import build as _B from semantic_model import loader as _L - org = _L.load_organization(profile_dir, include_rejected=True) + org = _L.load_datasource(profile_dir, include_rejected=True) storage_type = (org.storage_connections[0].storage_type if getattr(org, "storage_connections", None) else "") out_schemas: list[dict] = [] @@ -66,17 +66,17 @@ def build_manifest(profile_dir: Path, profile: str) -> dict: examples_out: list[dict] = [] areas_out: list[dict] = [] - org_md_path = profile_dir / "datasource.md" - organization_md = org_md_path.read_text(encoding="utf-8") if org_md_path.exists() else "" + datasource_md_path = profile_dir / "datasource.md" + datasource_md = datasource_md_path.read_text(encoding="utf-8") if datasource_md_path.exists() else "" import re as _re from semantic_model import org_draft as _OD - # `organization_md` is the human's narrative ONLY — this is what the edit box writes back, + # `datasource_md` is the human's narrative ONLY — this is what the edit box writes back, # so model facts must never be folded in here (or saving would persist them). If blank, # offer the starter prompt. The model-derived summary (subject areas, conventions, decoded # glossary) is a SEPARATE read-only field, computed fresh — see `derived_context`. - if not _re.sub(r"", "", organization_md, flags=_re.DOTALL).strip(): - organization_md = _OD.starter_organization_md(org) + if not _re.sub(r"", "", datasource_md, flags=_re.DOTALL).strip(): + datasource_md = _OD.starter_datasource_md(org) # Read-only derived block WITHOUT the curated glossary — that's rendered as an editable # panel (the curated key_terminology is a first-class structured field users add/correct). derived_context_md = _OD.derived_context(org, with_curated_glossary=False) @@ -218,7 +218,7 @@ def build_manifest(profile_dir: Path, profile: str) -> dict: return { "profile": profile, - "organization_md": organization_md, + "datasource_md": datasource_md, # model-derived domain summary (read-only in the UI; not part of the editable file) "derived_context_md": derived_context_md, # the curated glossary (term → definition) — EDITABLE in the explorer, written back @@ -285,7 +285,7 @@ def main() -> int: p.add_argument("--manifest-out", help="Optional: also dump the raw manifest JSON to this path") p.add_argument("--initial-tab", default="auto", - choices=["auto", "organization", "tables", "metrics", "entities", + choices=["auto", "datasource", "tables", "metrics", "entities", "joins", "examples", "review", "queued"], help="Tab the dashboard opens on. 'auto' (default) opens on Review when " "anything needs sign-off, else Tables; 'review' forces the sign-off queue.") diff --git a/plugins/agami/shared/model-explorer-template.html b/plugins/agami/shared/model-explorer-template.html index 7ac879cc..7699e665 100644 --- a/plugins/agami/shared/model-explorer-template.html +++ b/plugins/agami/shared/model-explorer-template.html @@ -24,7 +24,7 @@ { "profile": "main", "storage_type": "Snowflake", - "organization_md": "# About this database ...", + "datasource_md": "# About this database ...", "totals": { "schemas": 1, "tables": 14, "fields": 312, "metrics": 4, "entities": 3, "relationships": 9, "examples": 0, "validated": true }, "schemas": [ { "name": "sales", "description": "...", "tables": [ ...table... ] } ], @@ -404,7 +404,7 @@ pre.code-block.code-editable.empty { font-style: italic; color: var(--muted); opacity: 0.7; } pre.code-block.code-editable.edited { box-shadow: inset 3px 0 0 var(--accent); } - /* Organization markdown rendering */ + /* Datasource markdown rendering */ .org-rendered { padding: 4px 18px 14px; font-size: 14px; } .org-rendered h1 { font-size: 20px; margin: 16px 0 8px; } .org-rendered h2 { font-size: 16px; margin: 18px 0 6px; padding-bottom: 4px; border-bottom: 1px solid var(--border); } @@ -687,15 +687,15 @@

Send to Claude

// exampleEdits[qname] = {area, question, sql, source, status} // newMetrics[] = {area, name, description, calculation, binding, unit, other_names[], source_tables[]} // newExamples[] = {area, question, sql} (human-added NL→SQL examples) - // orgEdit = new datasource.md text | null + // dsEdit = new datasource.md text | null const pending = {}; const ops = {}; const edits = {}; const exampleEdits = {}; const newMetrics = []; const newExamples = []; - let orgEdit = null; - let orgRaw = false; // datasource.md editor mode: false = rich text, true = raw Markdown + let dsEdit = null; + let dsRaw = false; // datasource.md editor mode: false = rich text, true = raw Markdown let termEdits = null; // edited curated glossary {term: def} | null — written back via set-terminology const editOpen = new Set(); // qnames whose inline edit form is expanded @@ -734,7 +734,7 @@

Send to Claude

}; const TABS = [ - { id: 'organization', label: 'Organization' }, + { id: 'datasource', label: 'Datasource' }, { id: 'review', label: 'Review' }, { id: 'tables', label: 'Tables' }, { id: 'pii', label: 'PII' }, @@ -2382,7 +2382,7 @@

Send to Claude

} // ===================================================================== - // TAB: Organization + // TAB: Datasource // ===================================================================== function renderMarkdown(md) { const lines = (md || '').replace(/\r\n/g, '\n').split('\n'); @@ -2457,8 +2457,8 @@

Send to Claude

})(rootEl); return blocks.join('\n\n').replace(/\n{3,}/g, '\n\n').trim(); } - function renderOrganizationTab(root) { - const md = orgEdit !== null ? orgEdit : (manifest.organization_md || ''); + function renderDatasourceTab(root) { + const md = dsEdit !== null ? dsEdit : (manifest.datasource_md || ''); root.appendChild(el('div', { class: 'info-banner', html: 'What is this? This note tells agami about your business — what the company or product is, ' + 'what your words mean, and how your numbers are defined (for example, what counts as an "active customer"). ' + @@ -2473,14 +2473,14 @@

Send to Claude

const ORG = '__org__'; if (editOpen.has(ORG)) { const wrap = el('div', { class: 'edit-form', style: 'border-top:none;' }); - if (orgRaw) { + if (dsRaw) { // Power-user escape hatch: edit the raw Markdown directly. wrap.appendChild(labelLine('datasource.md (raw Markdown)')); const ta = textarea(md, 22); ta.style.minHeight = '360px'; wrap.appendChild(ta); const bar = el('div', { class: 'card-actions' }); - bar.appendChild(btnSm('Save', () => { orgEdit = ta.value; editOpen.delete(ORG); renderAll(); }, 'primary')); + bar.appendChild(btnSm('Save', () => { dsEdit = ta.value; editOpen.delete(ORG); renderAll(); }, 'primary')); bar.appendChild(btnSm('Cancel', () => { editOpen.delete(ORG); renderAll(); })); - bar.appendChild(btnSm('↩ Back to rich text', () => { orgEdit = ta.value; orgRaw = false; renderAll(); })); + bar.appendChild(btnSm('↩ Back to rich text', () => { dsEdit = ta.value; dsRaw = false; renderAll(); })); wrap.appendChild(bar); } else { // Rich-text editor: edit the formatted text directly — no Markdown symbols. @@ -2497,17 +2497,17 @@

Send to Claude

wrap.appendChild(tb); wrap.appendChild(ed); const bar = el('div', { class: 'card-actions' }); - bar.appendChild(btnSm('Save', () => { orgEdit = htmlToMarkdown(ed); editOpen.delete(ORG); renderAll(); }, 'primary')); + bar.appendChild(btnSm('Save', () => { dsEdit = htmlToMarkdown(ed); editOpen.delete(ORG); renderAll(); }, 'primary')); bar.appendChild(btnSm('Cancel', () => { editOpen.delete(ORG); renderAll(); })); - bar.appendChild(btnSm('⚙ Edit raw Markdown', () => { orgEdit = htmlToMarkdown(ed); orgRaw = true; renderAll(); })); + bar.appendChild(btnSm('⚙ Edit raw Markdown', () => { dsEdit = htmlToMarkdown(ed); dsRaw = true; renderAll(); })); wrap.appendChild(bar); } card.appendChild(wrap); } else { card.appendChild(el('div', { class: 'org-rendered', html: renderMarkdown(md) })); const a = el('div', { class: 'card-actions' }); - a.appendChild(el('button', { type: 'button', class: orgEdit !== null ? 'queued-edit' : '', - onclick: () => { editOpen.add(ORG); renderAll(); } }, orgEdit !== null ? '✎ Edited (queued) — edit again' : 'Edit datasource.md')); + a.appendChild(el('button', { type: 'button', class: dsEdit !== null ? 'queued-edit' : '', + onclick: () => { editOpen.add(ORG); renderAll(); } }, dsEdit !== null ? '✎ Edited (queued) — edit again' : 'Edit datasource.md')); card.appendChild(a); } root.appendChild(card); @@ -2638,9 +2638,9 @@

Send to Claude

groups.push(g); } // org edit - if (orgEdit !== null) { - const g = el('div', { class: 'queue-group' }); g.appendChild(el('h3', { text: 'Organization context' })); - g.appendChild(queueRow('edit', 'edit', 'datasource.md edited', () => { orgEdit = null; })); + if (dsEdit !== null) { + const g = el('div', { class: 'queue-group' }); g.appendChild(el('h3', { text: 'Datasource context' })); + g.appendChild(queueRow('edit', 'edit', 'datasource.md edited', () => { dsEdit = null; })); groups.push(g); } // glossary edit @@ -2746,7 +2746,7 @@

Send to Claude

// the In-schema/Cross-schema bar is Joins-tab only; renderJoinsTab decides whether to show it if (activeTab !== 'joins') document.getElementById('join-type-bar').style.display = 'none'; // search placeholder per tab - const ph = { organization: 'Search datasource.md…', review: 'Search the review queue…', + const ph = { datasource: 'Search datasource.md…', review: 'Search the review queue…', areas: 'Search subject areas…', tables: 'Search tables, fields, descriptions…', metrics: 'Search metrics…', entities: 'Search entities…', joins: 'Search joins…', @@ -2771,7 +2771,7 @@

Send to Claude

// ---- Queued totals + footer ---- function totalQueued() { return Object.keys(pending).length + Object.keys(ops).length + Object.keys(edits).length - + Object.keys(exampleEdits).length + newMetrics.length + newExamples.length + (orgEdit !== null ? 1 : 0) + + Object.keys(exampleEdits).length + newMetrics.length + newExamples.length + (dsEdit !== null ? 1 : 0) + (termEdits !== null ? 1 : 0); } function updateFooter() { @@ -2801,7 +2801,7 @@

Send to Claude

if (Object.keys(exampleEdits).length) parts.push(`${Object.keys(exampleEdits).length} example`); if (newExamples.length) parts.push(`${newExamples.length} new example${newExamples.length === 1 ? '' : 's'}`); if (newMetrics.length) parts.push(`${newMetrics.length} new metric${newMetrics.length === 1 ? '' : 's'}`); - if (orgEdit !== null) parts.push('org context'); + if (dsEdit !== null) parts.push('org context'); if (termEdits !== null) parts.push('glossary'); sum.innerHTML = '' + total + ' queued · ' + parts.join(', '); gen.disabled = false; @@ -2876,7 +2876,7 @@

Send to Claude

lines.push(JSON.stringify(newExamples.map(x => ({ area: x.area, question: x.question, sql: x.sql, source: 'human', status: 'confirmed' })))); } - if (orgEdit !== null) lines.push('organization-md: ' + JSON.stringify(orgEdit)); + if (dsEdit !== null) lines.push('datasource-md: ' + JSON.stringify(dsEdit)); if (termEdits !== null) lines.push('key-terminology: ' + JSON.stringify(termEdits)); // Sign-off identity header (first line) — the skill applies approvals with this as --signer/--role. if (hasQueuedApprovals() && validEmail(signOff.email)) lines.unshift(`signed-off-by: ${signOff.email} / ${signOffRole()}`); @@ -2905,7 +2905,7 @@

Send to Claude

Object.keys(ops).forEach(k => delete ops[k]); Object.keys(edits).forEach(k => delete edits[k]); Object.keys(exampleEdits).forEach(k => delete exampleEdits[k]); - newMetrics.length = 0; newExamples.length = 0; orgEdit = null; termEdits = null; addMetricOpen = false; addExampleOpen = false; + newMetrics.length = 0; newExamples.length = 0; dsEdit = null; termEdits = null; addMetricOpen = false; addExampleOpen = false; editOpen.clear(); renderAll(); }); @@ -2933,7 +2933,7 @@

Send to Claude

const root = document.getElementById('tab-content'); root.innerHTML = ''; switch (activeTab) { - case 'organization': renderOrganizationTab(root); break; + case 'datasource': renderDatasourceTab(root); break; case 'review': renderReviewTab(root); break; case 'areas': renderAreasTab(root); break; case 'tables': renderTablesTab(root); break; diff --git a/plugins/agami/shared/organization-context-format.md b/plugins/agami/shared/organization-context-format.md index 49512c81..339b02ed 100644 --- a/plugins/agami/shared/organization-context-format.md +++ b/plugins/agami/shared/organization-context-format.md @@ -70,7 +70,7 @@ The user replaces the comment with real prose, or leaves it — agami works eith ## When the skill loads it -- **`query-database`** — Phase 1d.2 runs `cli org-context`, injecting the combined narrative + derived summary under `## Organization context`, ahead of `## User memory`. +- **`query-database`** — Phase 1d.2 runs `cli org-context`, injecting the combined narrative + derived summary under `## Datasource context`, ahead of `## User memory`. - **`connect`** — enrichment uses the narrative (if any) as a domain prior for descriptions; writes the glossary to `key_terminology`; writes the starter on the skip path. If there's no narrative and no model, the context is empty — never an error. diff --git a/plugins/agami/shared/sql-generation-rules.md b/plugins/agami/shared/sql-generation-rules.md index a5d12160..b810370c 100644 --- a/plugins/agami/shared/sql-generation-rules.md +++ b/plugins/agami/shared/sql-generation-rules.md @@ -127,7 +127,7 @@ For growth rates (QoQ, MoM, YoY): ## Safety Rules -- **Result-set size policy** — do NOT auto-append `LIMIT 1000` (or any other implicit cap). Queries run unbounded by default. The query-database skill applies a per-org `max_rows` cap (configured in `/local/.config.json`) when present, and a per-query "top N" cap when the user explicitly asks for "top N" / "first N" / "limit to N". For all other queries, return the full result set. See [query SKILL.md → Result-set size policy](../skills/agami-query/SKILL.md#result-set-size-policy) for the canonical contract. +- **Result-set size policy** — do NOT auto-append `LIMIT 1000` (or any other implicit cap). Queries run unbounded by default. The query-database skill applies a per-org `max_rows` cap (configured in `/local/.config.json`) when present, and a per-query "top N" cap when the user explicitly asks for "top N" / "first N" / "limit to N". For all other queries, return the full result set. See [query SKILL.md → Result-set size policy](../skills/agami-query/SKILL.md#result-set-size-policy) for the canonical contract. - Never generate `DROP`, `DELETE`, `TRUNCATE`, `ALTER`, `INSERT`, `UPDATE`, or `CREATE` statements - Never include actual credential values in SQL comments or strings - Use `NULLIF(denominator, 0)` to guard against division by zero diff --git a/plugins/agami/skills/agami-connect/SKILL.md b/plugins/agami/skills/agami-connect/SKILL.md index dff92d4d..e2d2cefb 100644 --- a/plugins/agami/skills/agami-connect/SKILL.md +++ b/plugins/agami/skills/agami-connect/SKILL.md @@ -477,7 +477,7 @@ This is the union-rescan that makes Case 2 work: adding `billing` to a model tha Example to say BEFORE introspecting: *"Found 70 tables across 18 schemas. I'll model the **52** in the 16 ServiceNow module schemas. **Excluding:** `public` (17 — `accounts`/`opportunities`/`leads`… looks like a separate Salesforce dataset; onboard it as its own profile if you want it) and `pg_auto_copy` (1 — Redshift system). Good?"* A user who said "all" must SEE the delta up front — never discover later that 19 tables never made it in. This is mandatory whenever discovered-count ≠ scanned-count. -### 1.4 — Organization context (MANDATORY — ALWAYS ASK) +### 1.4 — Datasource context (MANDATORY — ALWAYS ASK) Two levels: a **company** description written **once** at the deployment root and shared by every datasource, and a **per-database** narrative for the datasource being connected now. The skill never decides yes/skip for the user; "don't ask clarifying questions" does NOT cancel this — it's required state-gathering. Ask **A** then **B**. diff --git a/plugins/agami/skills/agami-model/SKILL.md b/plugins/agami/skills/agami-model/SKILL.md index 3f1bfec4..505ad452 100644 --- a/plugins/agami/skills/agami-model/SKILL.md +++ b/plugins/agami/skills/agami-model/SKILL.md @@ -11,7 +11,7 @@ You are running the unified **model + trust** surface — one dashboard to brows This skill orchestrates: -1. **Render** — invoke `render_model_explorer.py` to walk every YAML and write a self-contained HTML artifact at `/local/model//.html`. The Python script does the YAML reading — **no LLM tokens spent on the walk**. The dashboard has tabs: **Organization · Review · Subject areas · Tables · Metrics · Entities · Joins · Examples · Queued**. The **Review** tab is the sign-off queue (the old `/agami-review`); pass `--initial-tab review` to open on it. +1. **Render** — invoke `render_model_explorer.py` to walk every YAML and write a self-contained HTML artifact at `/local/model//.html`. The Python script does the YAML reading — **no LLM tokens spent on the walk**. The dashboard has tabs: **Datasource · Review · Subject areas · Tables · Metrics · Entities · Joins · Examples · Queued**. The **Review** tab is the sign-off queue (the old `/agami-review`); pass `--initial-tab review` to open on it. 2. **Open + wait** — auto-open the file, end the turn, wait for the user to come back with a "Generate feedback for Claude" block (exclude/include, approve/reject, edits, new metrics, org edit). 3. **Apply** — for each batch, run `semantic_model.cli curate` with an ops JSON. The engine flips review_state / stamps sign-off, runs the validator, reverts via git on failure, appends to `curation_log.jsonl`, and commits. 4. **Re-render** — render to a new timestamped file and re-open. Wait for the next batch. @@ -74,7 +74,7 @@ Set `initial_tab=review` when `$ARGUMENTS` was `review`/`preseed`/`rule1` (Phase Model dashboard rendered — schema(s) · tables · fields · to review. /local/model//.html -Tabs: Organization · Review · Subject areas · Tables · Metrics · Entities · +Tabs: Datasource · Review · Subject areas · Tables · Metrics · Entities · Joins · Examples · Queued. Live search + status filters per tab. • Review tab — the sign-off queue: Approve / Reject the metrics (must be signed off before queries use them), entities, and inferred joins, with a @@ -107,7 +107,7 @@ End the turn. Wait for the user. ## Phase 2: Parse the chat back-channel -The user pastes the dashboard's "Generate feedback" block. **Don't hand-parse it** — pipe it to the parser, which handles the whole grammar (the `profile:` line, the `exclude/include tables|columns:` lists, and the `curate-ops` / `new-metrics` / `new-examples` / `example-edits` / `key-terminology` / `organization-md` JSON blocks) and translates the exclude/include lists into curate ops: +The user pastes the dashboard's "Generate feedback" block. **Don't hand-parse it** — pipe it to the parser, which handles the whole grammar (the `profile:` line, the `exclude/include tables|columns:` lists, and the `curate-ops` / `new-metrics` / `new-examples` / `example-edits` / `key-terminology` / `datasource-md` JSON blocks) and translates the exclude/include lists into curate ops: ```bash parse_model_feedback.py --block-file @@ -119,7 +119,7 @@ It prints `{data, anomalies, needs_judgment}`: - `data.new_metrics_by_area` — `{area: [metric…]}` → one `sm add "$ROOT" --kind metric --area --file ` per area. - `data.examples_by_area` — edited + new NL→SQL examples `{area: [example…]}` → one `sm add-example "$ROOT" --area --file ` per area (dedups by question). - `data.key_terminology` — the complete glossary object → `sm set-terminology "$ROOT" --file --replace`. -- `data.organization_md` — the full datasource.md text (already decoded) → **Write** it to `//datasource.md` (overwrite; human narrative only). +- `data.datasource_md` — the full datasource.md text (already decoded) → **Write** it to `//datasource.md` (overwrite; human narrative only). - `data.signer` / `data.role` — the curator identity for `--signer`/`--role` on every approve. **Persist** them into `/local/.config` (`reviewer_email`/`reviewer_role`; preserve other keys, `chmod 600`) so future sessions don't re-ask; if absent on an approve-bearing batch, fall back to `.config`, then the Phase 0 ask. **If `needs_judgment` is set, stop and ask** — `malformed_targets` (a table without its `.` prefix → *"Tables need a schema prefix — `sales.STG_LEADS`, not `STG_LEADS`. Did you mean that?"*) or `unparseable_json` (re-copy that block from the dashboard). Don't apply a partial block. diff --git a/plugins/agami/skills/agami-query/SKILL.md b/plugins/agami/skills/agami-query/SKILL.md index b579bdf7..cdccdfe9 100644 --- a/plugins/agami/skills/agami-query/SKILL.md +++ b/plugins/agami/skills/agami-query/SKILL.md @@ -125,11 +125,11 @@ This file holds free-form **user preferences across every database** (default fi Run `cli org-context "$ROOT"` — it returns the **full** domain context for this database in one block: the human's datasource.md narrative (HTML comments stripped) **plus** the model-derived summary that the file does NOT contain — subject areas, conventions, and the **decoded glossary** (`key_terminology` + enum legends), assembled fresh from the structured model. Don't `Read` datasource.md by hand: the file holds only the human narrative; the glossary and summary live in the model, and this command is the one that combines them. If there's no model, treat as empty — never error. See [`shared/organization-context-format.md`](../../shared/organization-context-format.md). -Inject the result into the SQL-generation prompt in Phase 2b under `## Organization context`, **before** the `## User memory` section — domain knowledge precedes display preferences in the LLM's reading order. +Inject the result into the SQL-generation prompt in Phase 2b under `## Datasource context`, **before** the `## User memory` section — domain knowledge precedes display preferences in the LLM's reading order. Order in Phase 2b prompt: 1. Schema context (tables / columns / relationships / metrics from the semantic model) -2. `## Organization context` ← from `cli org-context` (narrative + derived summary + glossary) +2. `## Datasource context` ← from `cli org-context` (narrative + derived summary + glossary) 3. `## User memory (preferences and policies)` ← from USER_MEMORY.md 4. Few-shot examples 5. The user's question @@ -225,7 +225,7 @@ For a single profile, follow the **examples-first canonical loop** — the subje 2. **Schema context** — the `get_table_context` output for the chosen tables (columns + types + caveats + value_transforms), the area's relationships (rendered as `from.col → to.col [cardinality]`), and the area's metrics (`: -- ` + synonyms). `default_filters` need not be enumerated — `execute_sql` auto-applies them (step below) — but DO honor any caveats. **Unreviewed metrics are USED, not refused.** When the question names a metric whose `review_state ≠ approved`, still use its binding and answer — do NOT block or refuse on it. The trust layer surfaces it as a **warning on the receipt** (Phase 4e.iii.5: *"Used metric `X` which has not been signed off"*), not a hard gate. The loader already drops only `rejected` metrics; an `unreviewed`/`proposed` one is yours to use, with the warning carrying the honesty. (Same for unreviewed joins/entities and `stale` entries — warn, never refuse.) -3. **Organization context** — `datasource.md` (step 1d.2), heading `## Organization context`. Binding domain context. +3. **Datasource context** — `datasource.md` (step 1d.2), heading `## Datasource context`. Binding domain context. 4. **User memory** — `USER_MEMORY.md` (step 1d.1), heading `## User memory (preferences and policies)`. 5. **Few-shot examples** — the ranked matches from step 2. 6. **User question.** diff --git a/tests/test_ace046_incremental_curation.py b/tests/test_ace046_incremental_curation.py index 3efbef76..b343023d 100644 --- a/tests/test_ace046_incremental_curation.py +++ b/tests/test_ace046_incremental_curation.py @@ -47,8 +47,8 @@ def _area_with_n_rels(name: str, n: int) -> m.SubjectArea: return m.SubjectArea(name=name, tables=refs, tables_defined=[a, b], relationships=rels) -def _org(*areas) -> m.Organization: - return m.Organization(datasource="O", +def _org(*areas) -> m.Datasource: + return m.Datasource(datasource="O", storage_connections=[m.StorageConnection(name="c", storage_type="PostgreSQL")], subject_areas=list(areas)) diff --git a/tests/test_ace047_schema_index.py b/tests/test_ace047_schema_index.py index 3f6322f4..a2240a5c 100644 --- a/tests/test_ace047_schema_index.py +++ b/tests/test_ace047_schema_index.py @@ -31,7 +31,7 @@ def _ref(table: str) -> m.TableRef: return m.TableRef(storage_connection="c", schema="public", table=table) -def _org_with_clash_and_fallback() -> m.Organization: +def _org_with_clash_and_fallback() -> m.Datasource: # alpha defines a table literally named "orders"; beta defines one named "sales.orders" (bare # "orders") — a name clash resolved by scan order. gamma REFERENCES orders (defined in alpha) # via a TableRef but doesn't define it — the multi-area fallback case. @@ -40,7 +40,7 @@ def _org_with_clash_and_fallback() -> m.Organization: tables_defined=[_tbl("sales.orders")]) gamma = m.SubjectArea(name="gamma", tables=[_ref("customers"), _ref("orders")], tables_defined=[_tbl("customers")]) - return m.Organization(datasource="O", + return m.Datasource(datasource="O", storage_connections=[m.StorageConnection(name="c", storage_type="PostgreSQL")], subject_areas=[alpha, beta, gamma]) @@ -67,7 +67,7 @@ def test_index_matches_linear_with_duplicate_area_names(): # returns the FIRST area of that name — so the index's per-area maps must be first-wins too. a1 = m.SubjectArea(name="sales", tables=[_ref("only_first")], tables_defined=[_tbl("only_first")]) a2 = m.SubjectArea(name="sales", tables=[_ref("only_second")], tables_defined=[_tbl("only_second")]) - org = m.Organization(datasource="O", + org = m.Datasource(datasource="O", storage_connections=[m.StorageConnection(name="c", storage_type="PostgreSQL")], subject_areas=[a1, a2]) idx = L.build_table_index(org) diff --git a/tests/test_ace051_fail_closed.py b/tests/test_ace051_fail_closed.py index baec5ca2..a3002465 100644 --- a/tests/test_ace051_fail_closed.py +++ b/tests/test_ace051_fail_closed.py @@ -24,12 +24,12 @@ from semantic_model import models as m # noqa: E402 -def _org() -> m.Organization: +def _org() -> m.Datasource: """A model declaring exactly two tables: orders, customers.""" def _t(name): return m.Table(name=name, schema="public", storage_connection="c", grain=["id"], description=name, columns=[m.Column(name="id", type="integer")]) - return m.Organization( + return m.Datasource( datasource="Shop", subject_areas=[m.SubjectArea(name="sales", tables_defined=[_t("orders"), _t("customers")])], ) @@ -41,7 +41,7 @@ def _seed_db(url: str, ds: str = "acme") -> None: s = Store.connect(url) s.run_migrations() - model_store.write_organization(s, ds, _org()) + model_store.write_datasource(s, ds, _org()) s.close() diff --git a/tests/test_admin_model.py b/tests/test_admin_model.py index 7d5e0b4f..07928314 100644 --- a/tests/test_admin_model.py +++ b/tests/test_admin_model.py @@ -1,6 +1,6 @@ """The admin read-only Model explorer. -A pure projection of the served model (`load_organization`) + domain docs (`load_memory`): the +A pure projection of the served model (`load_datasource`) + domain docs (`load_memory`): the overview / area landing / table page render the served tree; it is admin-gated, read-only (a GET only), never leaks `storage_config`, and escapes operator-authored text. """ @@ -26,7 +26,7 @@ import model_store # noqa: E402 import ui # noqa: E402 import user_store # noqa: E402 -from semantic_model.models import Organization # noqa: E402 +from semantic_model.models import Datasource # noqa: E402 from starlette.testclient import TestClient # noqa: E402 from store import Store # noqa: E402 @@ -193,7 +193,7 @@ def _seed(url, datasource="SALES_DATA", org=ORG): s = Store.connect(url) s.run_migrations() - model_store.write_organization(s, datasource, Organization.model_validate(org)) + model_store.write_datasource(s, datasource, Datasource.model_validate(org)) s.close() diff --git a/tests/test_aggregation_class.py b/tests/test_aggregation_class.py index 65c6416d..626fda92 100644 --- a/tests/test_aggregation_class.py +++ b/tests/test_aggregation_class.py @@ -109,7 +109,7 @@ def test_curator_can_edit_aggregation(tmp_path): "column": "amount", "field": "aggregation", "value": "averageable", }]) assert not res.errors, res.errors - reloaded = __import__("semantic_model.loader", fromlist=["load_organization"]).load_organization(root) + reloaded = __import__("semantic_model.loader", fromlist=["load_datasource"]).load_datasource(root) t = reloaded.subject_areas[0].defined_table("orders") assert t.get_column("amount").aggregation == "averageable" diff --git a/tests/test_aggregation_enforcement.py b/tests/test_aggregation_enforcement.py index b54d581e..4c534ffe 100644 --- a/tests/test_aggregation_enforcement.py +++ b/tests/test_aggregation_enforcement.py @@ -30,7 +30,7 @@ def _org(columns, metrics=None, table="facts"): columns=columns) sa = m.SubjectArea(name="area", description="d", tables_defined=[t], metrics=metrics or []) - return m.Organization(datasource="o", version=1, subject_areas=[sa]) + return m.Datasource(datasource="o", version=1, subject_areas=[sa]) # --- #2 aggregation-class enforcement -------------------------------------- @@ -124,7 +124,7 @@ def test_semi_additive_is_table_scoped_no_cross_table_misfire(): bindings={"PostgreSQL": "SUM(balance)"}, source_tables=["facts"], non_additive_dimensions=["time"], semi_additive_agg="last") sa = m.SubjectArea(name="area", description="d", tables_defined=[facts, ledger], metrics=[semi]) - org = m.Organization(datasource="o", version=1, subject_areas=[sa]) + org = m.Datasource(datasource="o", version=1, subject_areas=[sa]) # ledger.balance is NOT the semi-additive one → allowed r = RT.pre_flight_check("SELECT entry_date, SUM(balance) FROM ledger GROUP BY entry_date", org) assert r.action == "allow" diff --git a/tests/test_choice_field_decode.py b/tests/test_choice_field_decode.py index 970ac4e6..c61dc48e 100644 --- a/tests/test_choice_field_decode.py +++ b/tests/test_choice_field_decode.py @@ -64,7 +64,7 @@ def test_catalog_coded_column_gets_choice_skeleton(tmp_path): def test_unlabeled_choice_fields_reports_then_clears(tmp_path): org, _ = I.introspect("sn", "postgres", runner=_catalog_runner, artifacts_dir=tmp_path) root = tmp_path / "sn" - rep = C.unlabeled_choice_fields(__import__("semantic_model.loader", fromlist=["load_organization"]).load_organization(root, include_rejected=True)) + rep = C.unlabeled_choice_fields(__import__("semantic_model.loader", fromlist=["load_datasource"]).load_datasource(root, include_rejected=True)) assert rep["count"] == 1 and rep["ok"] is False assert rep["unlabeled"][0]["column"] == "severity" @@ -74,6 +74,6 @@ def test_unlabeled_choice_fields_reports_then_clears(tmp_path): "column": "severity", "field": "choice_field", "value": {"1": "High", "2": "Medium", "3": "Low"}}]) assert not res.errors, res.errors - org2 = __import__("semantic_model.loader", fromlist=["load_organization"]).load_organization(root) + org2 = __import__("semantic_model.loader", fromlist=["load_datasource"]).load_datasource(root) assert org2.subject_areas[0].defined_table("incident").get_column("severity").choice_field == {"1": "High", "2": "Medium", "3": "Low"} assert C.unlabeled_choice_fields(org2)["ok"] is True diff --git a/tests/test_cloud_neutral_config.py b/tests/test_cloud_neutral_config.py index 1b5f9746..bebd002a 100644 --- a/tests/test_cloud_neutral_config.py +++ b/tests/test_cloud_neutral_config.py @@ -177,16 +177,16 @@ def test_db_backed_serve_needs_no_artifacts_dir(tmp_path, monkeypatch): import model_store import tools - from semantic_model.models import Organization + from semantic_model.models import Datasource from store import Store db_url = "sqlite://" + str(tmp_path / "agami.db") s = Store.connect(db_url) s.run_migrations() - model_store.write_organization( + model_store.write_datasource( s, "main", - Organization.model_validate( + Datasource.model_validate( { "datasource": "acme", "version": 1, diff --git a/tests/test_column_scope_adversarial.py b/tests/test_column_scope_adversarial.py index dbfd1a40..46a70d45 100644 --- a/tests/test_column_scope_adversarial.py +++ b/tests/test_column_scope_adversarial.py @@ -40,7 +40,7 @@ def _t(name, cols): orders = _t("orders", [("id", "integer"), ("amount", "decimal"), ("customer_id", "integer"), ("status", "string")]) customers = _t("customers", [("id", "integer"), ("name", "string"), ("region", "string")]) - return m.Organization(datasource="Shop", + return m.Datasource(datasource="Shop", subject_areas=[m.SubjectArea(name="sales", tables_defined=[orders, customers])]) diff --git a/tests/test_column_scope_gate.py b/tests/test_column_scope_gate.py index 3fb221e3..9b2e46d8 100644 --- a/tests/test_column_scope_gate.py +++ b/tests/test_column_scope_gate.py @@ -35,7 +35,7 @@ def _t(name, cols): orders = _t("orders", [("id", "integer"), ("amount", "decimal"), ("customer_id", "integer"), ("status", "string")]) customers = _t("customers", [("id", "integer"), ("name", "string"), ("region", "string")]) - return m.Organization(datasource="Shop", + return m.Datasource(datasource="Shop", subject_areas=[m.SubjectArea(name="sales", tables_defined=[orders, customers])]) @@ -154,7 +154,7 @@ def test_case_insensitive_match(): # --- column-scope: degrade-to-allow ---------------------------------------- def test_column_empty_model_allows(): - org = m.Organization(datasource="Empty", subject_areas=[m.SubjectArea(name="s")]) + org = m.Datasource(datasource="Empty", subject_areas=[m.SubjectArea(name="s")]) assert rt.check_column_scope("SELECT anything FROM whatever", org).action == "allow" diff --git a/tests/test_compose_org_context.py b/tests/test_compose_org_context.py index 6339aee1..46eef4f4 100644 --- a/tests/test_compose_org_context.py +++ b/tests/test_compose_org_context.py @@ -20,15 +20,15 @@ from semantic_model import org_draft as OD # noqa: E402 from semantic_model.models import ( # noqa: E402 + Datasource, DisplayConventions, - Organization, OrgRecord, SubjectArea, ) -def _org(name: str, area: str, account_means: str) -> Organization: - return Organization( +def _org(name: str, area: str, account_means: str) -> Datasource: + return Datasource( datasource=name, subject_areas=[SubjectArea(name=area, description=f"{area} area")], key_terminology={"Account": account_means}, diff --git a/tests/test_coverage_gate_self_evident.py b/tests/test_coverage_gate_self_evident.py index ac458587..023cb6ac 100644 --- a/tests/test_coverage_gate_self_evident.py +++ b/tests/test_coverage_gate_self_evident.py @@ -34,7 +34,7 @@ def _incident_cols(): def _org(tables): - return m.Organization(datasource="o", + return m.Datasource(datasource="o", subject_areas=[m.SubjectArea(name="s", tables_defined=tables)]) diff --git a/tests/test_coverage_underenriched.py b/tests/test_coverage_underenriched.py index d6da2ab2..6012bb0b 100644 --- a/tests/test_coverage_underenriched.py +++ b/tests/test_coverage_underenriched.py @@ -28,7 +28,7 @@ def _col(name, desc=""): def _org(tables): sa = m.SubjectArea(name="a", description="d", tables_defined=tables) - return m.Organization(datasource="o", version=1, subject_areas=[sa]) + return m.Datasource(datasource="o", version=1, subject_areas=[sa]) def test_self_evident_blanks_are_fine(): diff --git a/tests/test_derived_metrics.py b/tests/test_derived_metrics.py index a639d83a..bf029984 100644 --- a/tests/test_derived_metrics.py +++ b/tests/test_derived_metrics.py @@ -31,7 +31,7 @@ def _metric(name, binding, *, base=None, calc="x", tables=("orders",)): def _org(metrics): sa = m.SubjectArea(name="sales", description="d", metrics=metrics) - return m.Organization(datasource="o", version=1, subject_areas=[sa]) + return m.Datasource(datasource="o", version=1, subject_areas=[sa]) # --- resolver --------------------------------------------------------------- diff --git a/tests/test_guard_context.py b/tests/test_guard_context.py index ede52a5c..285db2fe 100644 --- a/tests/test_guard_context.py +++ b/tests/test_guard_context.py @@ -40,7 +40,7 @@ def _org(): rels = [m.Relationship(from_table="orders", to_table="customers", from_column="customer_id", to_column="id", relationship="many_to_one")] - return m.Organization( + return m.Datasource( datasource="Shop", subject_areas=[m.SubjectArea(name="sales", tables_defined=[customers, orders], relationships=rels)], diff --git a/tests/test_mcp_harness.py b/tests/test_mcp_harness.py index ab3538e8..04f9e01f 100644 --- a/tests/test_mcp_harness.py +++ b/tests/test_mcp_harness.py @@ -332,7 +332,7 @@ def test_mcp_receipt_equals_shared_assembler(monkeypatch, tmp_path): monkeypatch.setenv("AGAMI_ARTIFACTS_DIR", str(art)) from semantic_model import loader as L from semantic_model import runtime as RT - shared = RT.assemble_receipt(L.load_organization(root), SQL) + shared = RT.assemble_receipt(L.load_datasource(root), SQL) mcp = _resolve_receipt(profile, SQL) for key in ("tables_used", "relationships", "metrics", "assumptions", "warnings"): assert mcp[key] == shared[key], key diff --git a/tests/test_metadata_sources.py b/tests/test_metadata_sources.py index e0ed70b7..f2b068ee 100644 --- a/tests/test_metadata_sources.py +++ b/tests/test_metadata_sources.py @@ -151,7 +151,7 @@ def test_generated_ops_apply_and_stamp_metadata(tmp_path): assert res.validated and not res.errors, res.as_dict() assert len(res.applied) == 3 - org = loader.load_organization(tmp_path) + org = loader.load_datasource(tmp_path) incident = next(t for sa in org.subject_areas for t in sa.tables_defined if t.name == "incident") cols = {c.name: c for c in incident.columns} assert cols["severity"].choice_field == {"1": "High", "2": "Medium", "3": "Low"} @@ -177,7 +177,7 @@ def tbl(name, extra): tables_defined=[tbl("incident", ["assignment_group"]), tbl("problem", ["assignment_group"])]) sysa = mm.SubjectArea(name="sys", description="d", tables_defined=[tbl("sys_user_group", [])]) - org = mm.Organization(datasource="o", version=1, subject_areas=[itsm, sysa]) + org = mm.Datasource(datasource="o", version=1, subject_areas=[itsm, sysa]) specs = [{"from_table": t.name, "from_column": c.name, "to_table": decls[c.name.lower()]} for sa in org.subject_areas for t in sa.tables_defined for c in t.columns if c.name.lower() in decls] @@ -195,7 +195,7 @@ def tbl(name): columns=[mm.Column(name="id", type="integer", primary_key=True)]) itsm = mm.SubjectArea(name="itsm", description="d", tables_defined=[tbl("incident"), tbl("problem")]) sysa = mm.SubjectArea(name="sys", description="d", tables_defined=[tbl("sys_user")]) - org = mm.Organization(datasource="o", version=1, subject_areas=[itsm, sysa]) + org = mm.Datasource(datasource="o", version=1, subject_areas=[itsm, sysa]) specs = [ {"from_table": "incident", "from_column": "problem_id", "to_table": "problem"}, # intra (itsm) {"from_table": "incident", "from_column": "caller_id", "to_table": "sys_user"}, # cross (itsm→sys) @@ -278,7 +278,7 @@ def run(sql): ["enrich-metadata", str(tmp_path), "--profile", "servicenow", "--db-type", "redshift"]) assert ns.func(ns) == 0 - org = loader.load_organization(tmp_path) + org = loader.load_datasource(tmp_path) incident = next(t for sa in org.subject_areas for t in sa.tables_defined if t.name == "incident") cols = {c.name: c for c in incident.columns} assert cols["severity"].choice_field == {"1": "High", "2": "Medium", "3": "Low"} @@ -322,7 +322,7 @@ def run(sql): ns = cli.build_parser().parse_args( ["enrich-metadata", str(tmp_path), "--profile", "servicenow", "--db-type", "redshift"]) assert ns.func(ns) == 0 - org = loader.load_organization(tmp_path) + org = loader.load_datasource(tmp_path) assert not [r for r in org.cross_subject_area_relationships if r.to_table == "sys_user"] @@ -342,7 +342,7 @@ def tbl(name, cols, perf=None): + [mm.Column(name=c, type="integer") for c in cols]) itsm = mm.SubjectArea(name="itsm", description="d", tables_defined=[tbl("incident", ["caller_id"], big)]) sysa = mm.SubjectArea(name="sys", description="d", tables_defined=[tbl("sys_user", [])]) - org = mm.Organization(datasource="o", version=1, subject_areas=[itsm, sysa]) + org = mm.Datasource(datasource="o", version=1, subject_areas=[itsm, sysa]) probed = {"n": 0} def runner(sql): diff --git a/tests/test_metric_binding_and_prune.py b/tests/test_metric_binding_and_prune.py index f97f928f..d9e2f239 100644 --- a/tests/test_metric_binding_and_prune.py +++ b/tests/test_metric_binding_and_prune.py @@ -23,7 +23,7 @@ def _org(metric): m.Column(name="id", type="integer", primary_key=True), m.Column(name="cost", type="decimal")]) sa = m.SubjectArea(name="s", tables_defined=[t], metrics=[metric]) - return m.Organization(datasource="t", subject_areas=[sa]) + return m.Datasource(datasource="t", subject_areas=[sa]) # --- binding column check --------------------------------------------------- diff --git a/tests/test_model_deploy.py b/tests/test_model_deploy.py index 1573f2d3..d47bfb8c 100644 --- a/tests/test_model_deploy.py +++ b/tests/test_model_deploy.py @@ -61,7 +61,7 @@ def test_deploy_loads_model_and_round_trips(tmp_path): _write_model(arts, "demo") store = _store(tmp_path) loaded = model_deploy.deploy_models(store, arts) - org = model_store.load_organization(store, "demo") + org = model_store.load_datasource(store, "demo") store.close() assert loaded == ["demo"] assert org is not None and org.datasource == "acme" @@ -158,7 +158,7 @@ def test_main_deploys_all_then_a_named_datasource(tmp_path, monkeypatch): assert model_deploy.main([]) == 0 # deploy every model under the dir (migrates first, then loads) assert model_deploy.main(["demo"]) == 0 # deploy a named datasource store = Store.connect("sqlite://" + str(tmp_path / "m.db")) - org = model_store.load_organization(store, "demo") + org = model_store.load_datasource(store, "demo") store.close() assert org is not None and org.datasource == "acme" @@ -181,7 +181,7 @@ def test_malformed_examples_file_is_skipped_not_fatal(tmp_path): ) store = _store(tmp_path) loaded = model_deploy.deploy_models(store, arts) # must not raise - org = model_store.load_organization(store, "demo") + org = model_store.load_datasource(store, "demo") n_examples = _count(store, "prompt_example", "demo") store.close() assert loaded == ["demo"] and org is not None # the model deployed despite the bad examples file diff --git a/tests/test_model_feedback.py b/tests/test_model_feedback.py index c668d288..93cc339e 100644 --- a/tests/test_model_feedback.py +++ b/tests/test_model_feedback.py @@ -79,11 +79,11 @@ def test_examples_orgmd_and_slash_signer(): "signed-off-by: bob@x.com / data_lead\n" 'example-edits: [{"area":"sales","question":"q1","sql":"SELECT 1"}]\n' 'new-examples: [{"area":"ops","question":"q2","sql":"SELECT 2"}]\n' - 'organization-md: "About this DB.\\nMRR = monthly recurring revenue."\n' + 'datasource-md: "About this DB.\\nMRR = monthly recurring revenue."\n' "done\n" ) data, _, needs = F.parse(block) assert needs is None assert (data["signer"], data["role"]) == ("bob@x.com", "data_lead") assert set(data["examples_by_area"]) == {"sales", "ops"} - assert data["organization_md"].startswith("About this DB.") + assert data["datasource_md"].startswith("About this DB.") diff --git a/tests/test_model_store_roundtrip.py b/tests/test_model_store_roundtrip.py index bcb6484b..0b60a61e 100644 --- a/tests/test_model_store_roundtrip.py +++ b/tests/test_model_store_roundtrip.py @@ -1,8 +1,8 @@ """Serve the model from the DB — golden parity with the file loader (Slice C). -Two proofs: (1) writing then loading an Organization through the DB is lossless for every object +Two proofs: (1) writing then loading an Datasource through the DB is lossless for every object type; (2) a model loaded from YAML files, seeded to the DB, and re-loaded from a *fresh* connection -yields the identical Organization — and tools._load_org / get_datasource_schema serve from the DB +yields the identical Datasource — and tools._load_org / get_datasource_schema serve from the DB (files absent) when AGAMI_DB_URL is set. """ @@ -17,7 +17,7 @@ import model_store # noqa: E402 import tools # noqa: E402 from semantic_model import loader as L # noqa: E402 -from semantic_model.models import Organization # noqa: E402 +from semantic_model.models import Datasource # noqa: E402 from store import Store # noqa: E402 FULL_ORG = { @@ -68,11 +68,11 @@ def test_db_roundtrip_is_lossless_for_every_object_type(): - org = Organization.model_validate(FULL_ORG) + org = Datasource.model_validate(FULL_ORG) s = Store.connect("sqlite://") s.run_migrations() - model_store.write_organization(s, "main", org) - rebuilt = model_store.load_organization(s, "main") + model_store.write_datasource(s, "main", org) + rebuilt = model_store.load_datasource(s, "main") assert rebuilt is not None assert rebuilt.model_dump(mode="json") == org.model_dump(mode="json") s.close() @@ -81,16 +81,16 @@ def test_db_roundtrip_is_lossless_for_every_object_type(): def test_load_missing_datasource_returns_none(): s = Store.connect("sqlite://") s.run_migrations() - assert model_store.load_organization(s, "nope") is None + assert model_store.load_datasource(s, "nope") is None s.close() def test_reseed_replaces_rows(): s = Store.connect("sqlite://") s.run_migrations() - org = Organization.model_validate(FULL_ORG) - model_store.write_organization(s, "main", org) - model_store.write_organization(s, "main", org) # idempotent re-seed, not a duplicate + org = Datasource.model_validate(FULL_ORG) + model_store.write_datasource(s, "main", org) + model_store.write_datasource(s, "main", org) # idempotent re-seed, not a duplicate assert len(s.query("SELECT name FROM subject_area WHERE datasource='main'")) == 1 s.close() @@ -157,18 +157,18 @@ def _write_file_model(root): def test_file_model_seeds_to_db_and_tools_serve_from_it(tmp_path, monkeypatch): art = tmp_path / "art" _write_file_model(art / "main") - file_org = L.load_organization(art / "main") + file_org = L.load_datasource(art / "main") db_url = "sqlite://" + str(tmp_path / "agami.db") s = Store.connect(db_url) s.run_migrations() - model_store.write_organization(s, "main", file_org) + model_store.write_datasource(s, "main", file_org) s.commit() s.close() - # a fresh connection (a "second instance") rebuilds the identical Organization + # a fresh connection (a "second instance") rebuilds the identical Datasource s2 = Store.connect(db_url) - db_org = model_store.load_organization(s2, "main") + db_org = model_store.load_datasource(s2, "main") s2.close() assert db_org.model_dump(mode="json") == file_org.model_dump(mode="json") @@ -220,15 +220,15 @@ def test_tools_serve_memory_and_version_from_db_no_files(tmp_path, monkeypatch): monkeypatch.setenv("AGAMI_DB_URL", db_url) monkeypatch.setenv("AGAMI_ARTIFACTS_DIR", str(tmp_path / "does-not-exist")) assert tools._model_version("main") == "v-deadbeef" - org_md, user_md, _, _ = tools._context_sources("main", tools._current_org_id()) - assert "Widgets co." in org_md and user_md == "exclude test users" + datasource_md, user_md, _, _ = tools._context_sources("main", tools._current_org_id()) + assert "Widgets co." in datasource_md and user_md == "exclude test users" def _seed_org(tmp_path, monkeypatch, org_dict) -> None: db_url = "sqlite://" + str(tmp_path / "agami.db") s = Store.connect(db_url) s.run_migrations() - model_store.write_organization(s, "main", Organization.model_validate(org_dict)) + model_store.write_datasource(s, "main", Datasource.model_validate(org_dict)) s.close() monkeypatch.setenv("AGAMI_DB_URL", db_url) monkeypatch.setenv("AGAMI_ARTIFACTS_DIR", str(tmp_path / "none")) diff --git a/tests/test_multi_area_membership.py b/tests/test_multi_area_membership.py index d40a8d5c..6d284263 100644 --- a/tests/test_multi_area_membership.py +++ b/tests/test_multi_area_membership.py @@ -30,7 +30,7 @@ def _two_area_org(): area_b = m.SubjectArea(name="incidents", tables_defined=[own], tables=[ m.TableRef(storage_connection="c", schema="s", table="incident"), m.TableRef(storage_connection="c", schema="s", table="sys_user")]) - return m.Organization(datasource="o", subject_areas=[area_a, area_b]) + return m.Datasource(datasource="o", subject_areas=[area_a, area_b]) def test_find_table_resolves_referenced_table_from_other_area(): diff --git a/tests/test_org_cache.py b/tests/test_org_cache.py index 6809ea97..ed44d576 100644 --- a/tests/test_org_cache.py +++ b/tests/test_org_cache.py @@ -29,7 +29,7 @@ def fake_load(profile): first = tools.get_cached_org("sales") second = tools.get_cached_org("sales") assert calls["load"] == 1 # query #2 served warm from the cache - assert first is second # the same cached Organization object + assert first is second # the same cached Datasource object def test_reloads_after_version_bump(monkeypatch): diff --git a/tests/test_org_draft.py b/tests/test_org_draft.py index 15ef076d..b8cce561 100644 --- a/tests/test_org_draft.py +++ b/tests/test_org_draft.py @@ -46,9 +46,9 @@ def _model(root): def test_derived_context_is_pure_summary(tmp_path): from semantic_model import org_draft - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) - d = org_draft.derived_context(load_organization(tmp_path)) + d = org_draft.derived_context(load_datasource(tmp_path)) # SUMMARY shape: counts + subject areas + conventions + glossary — NOT a model dump assert "**acme** — 1 table across 1 subject area" in d assert "### Subject areas" in d and "sales" in d and "orders & customers" in d @@ -63,14 +63,14 @@ def test_derived_context_can_exclude_curated_glossary(tmp_path): # the explorer renders the curated glossary as an EDITABLE panel, so it asks derived_context # to omit those terms — but the derived ENUM legends (from choice_field) still show read-only. from semantic_model import curate, org_draft - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) p = tmp_path / "subject_areas" / "s" / "tables" / "orders.yaml" d = yaml.safe_load(p.read_text()) d["columns"].append({"name": "status", "type": "string", "choice_field": {"P": "pending"}}) p.write_text(yaml.safe_dump(d)) curate.set_key_terminology(tmp_path, {"MRR": "monthly recurring revenue"}) - org = load_organization(tmp_path) + org = load_datasource(tmp_path) full = org_draft.derived_context(org) # LLM: curated + enum explorer = org_draft.derived_context(org, with_curated_glossary=False) # read-only: enum only assert "MRR" in full and "monthly recurring revenue" in full @@ -90,24 +90,24 @@ def test_explorer_exposes_glossary_as_editable_field(tmp_path): def test_compose_keeps_human_narrative_and_derived_facts_separate(tmp_path): from semantic_model import org_draft - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) human = "# About this database\n\nWe are a lending startup. MRR = monthly recurring revenue.\n" - md = org_draft.compose_context(human, load_organization(tmp_path)) + md = org_draft.compose_context(human, load_datasource(tmp_path)) # the human's words are preserved verbatim... assert "We are a lending startup." in md and "MRR = monthly recurring revenue." in md # ...and the derived facts sit under their OWN heading, never mixed into the prose assert "## Model summary (auto-generated from your schema)" in md assert "### Subject areas" in md and "INR" in md # empty human → derived only; empty model-less call → empty - assert "Model summary" in org_draft.compose_context("", load_organization(tmp_path)) + assert "Model summary" in org_draft.compose_context("", load_datasource(tmp_path)) def test_key_terminology_seeded_from_glossary_and_enums(tmp_path): # the section is no longer a bare prompt: curated glossary terms + auto-derived enum # legends from choice_field columns both render. from semantic_model import curate, org_draft - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) p = tmp_path / "subject_areas" / "s" / "tables" / "orders.yaml" doc = yaml.safe_load(p.read_text()) @@ -116,7 +116,7 @@ def test_key_terminology_seeded_from_glossary_and_enums(tmp_path): p.write_text(yaml.safe_dump(doc)) res = curate.set_key_terminology(tmp_path, {"MRR": "monthly recurring revenue", "ARR": "annual recurring revenue"}) assert res.validated and res.applied - md = org_draft.draft_organization_md(load_organization(tmp_path)) + md = org_draft.draft_datasource_md(load_datasource(tmp_path)) assert "**MRR** — monthly recurring revenue" in md assert "**ARR** — annual recurring revenue" in md assert "orders.status" in md and "`P` = pending" in md # auto enum legend @@ -125,14 +125,14 @@ def test_key_terminology_seeded_from_glossary_and_enums(tmp_path): def test_set_key_terminology_merges_then_replaces(tmp_path): from semantic_model import curate - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) assert curate.set_key_terminology(tmp_path, {"MRR": "monthly recurring revenue"}).validated curate.set_key_terminology(tmp_path, {"churn": "no order in 90 days"}) # merge (default) - assert load_organization(tmp_path).key_terminology == { + assert load_datasource(tmp_path).key_terminology == { "MRR": "monthly recurring revenue", "churn": "no order in 90 days"} curate.set_key_terminology(tmp_path, {"gold tier": "lifetime spend over 10k"}, merge=False) # replace - assert load_organization(tmp_path).key_terminology == {"gold tier": "lifetime spend over 10k"} + assert load_datasource(tmp_path).key_terminology == {"gold tier": "lifetime spend over 10k"} def test_explorer_org_md_is_human_only_derived_is_a_separate_field(tmp_path): @@ -140,16 +140,16 @@ def test_explorer_org_md_is_human_only_derived_is_a_separate_field(tmp_path): _model(tmp_path) # no datasource.md → editable field is the human STARTER (prompt only, NO facts)... m = build_manifest(tmp_path, "acme") - assert "About this database" in m["organization_md"] - assert "Subject areas" not in m["organization_md"] # facts never in the editable file + assert "About this database" in m["datasource_md"] + assert "Subject areas" not in m["datasource_md"] # facts never in the editable file # ...and the model-derived facts live in their own read-only field assert "Subject areas" in m["derived_context_md"] and "sales" in m["derived_context_md"] # a comments-only file is still "blank" → starter; facts stay separate (tmp_path / "datasource.md").write_text("\n") m2 = build_manifest(tmp_path, "acme") - assert "About this database" in m2["organization_md"] and "Subject areas" in m2["derived_context_md"] + assert "About this database" in m2["datasource_md"] and "Subject areas" in m2["derived_context_md"] # a real human file is left as-is in the editable field; facts still separate (tmp_path / "datasource.md").write_text("# About\nWe are a lending startup.") m3 = build_manifest(tmp_path, "acme") - assert m3["organization_md"] == "# About\nWe are a lending startup." + assert m3["datasource_md"] == "# About\nWe are a lending startup." assert "Subject areas" in m3["derived_context_md"] diff --git a/tests/test_org_identity_e2e.py b/tests/test_org_identity_e2e.py index d06e9821..6974ccf3 100644 --- a/tests/test_org_identity_e2e.py +++ b/tests/test_org_identity_e2e.py @@ -21,13 +21,13 @@ import tools # noqa: E402 from semantic_model import build, loader # noqa: E402 -from semantic_model.models import Organization # noqa: E402 +from semantic_model.models import Datasource # noqa: E402 _HEX = set("0123456789abcdef") -def _minimal_org(name: str = "acme") -> Organization: - return Organization(datasource=name) +def _minimal_org(name: str = "acme") -> Datasource: + return Datasource(datasource=name) def test_write_tree_mints_uuid4_and_is_immutable(tmp_path): @@ -38,7 +38,7 @@ def test_write_tree_mints_uuid4_and_is_immutable(tmp_path): assert minted and len(minted) == 32 and set(minted) <= _HEX # a uuid4 hex, minted locally # load -> mutate -> rewrite: the id is preserved, never re-minted (immutability guarantee). - org = loader.load_organization(prof) + org = loader.load_datasource(prof) assert org.org_id == minted org.description = "edited" build.write_tree(org, prof) @@ -99,7 +99,7 @@ def test_legacy_profile_without_org_id_resolves_local(tmp_path, monkeypatch): (tmp_path / "old").mkdir() (tmp_path / "old" / "datasource.yaml").write_text("datasource: legacy\nversion: 1\n") assert loader.load_org_id(tmp_path / "old") is None - assert loader.load_organization(tmp_path / "old").org_id is None + assert loader.load_datasource(tmp_path / "old").org_id is None monkeypatch.setenv("AGAMI_ARTIFACTS_DIR", str(tmp_path)) monkeypatch.delenv("AGAMI_ORG_ID", raising=False) diff --git a/tests/test_org_record.py b/tests/test_org_record.py index 53ce7f8e..f1372b6d 100644 --- a/tests/test_org_record.py +++ b/tests/test_org_record.py @@ -29,13 +29,13 @@ import tools # noqa: E402 from semantic_model import build, loader # noqa: E402 from semantic_model import org_record as OR # noqa: E402 -from semantic_model.models import DisplayConventions, Organization, OrgRecord # noqa: E402 +from semantic_model.models import Datasource, DisplayConventions, OrgRecord # noqa: E402 _HEX = set("0123456789abcdef") -def _minimal_org(name: str = "acme") -> Organization: - return Organization(datasource=name) +def _minimal_org(name: str = "acme") -> Datasource: + return Datasource(datasource=name) def test_ensure_org_record_mints_once_into_organization_yaml(tmp_path): @@ -129,7 +129,7 @@ def test_org_record_roundtrips_structured_content_losslessly(tmp_path): def test_bare_record_is_valid_and_fiscal_year_bounds_enforced(): assert OrgRecord(org_id="x").fiscal_year_start_month is None # id-only record validates with pytest.raises(ValueError): - OrgRecord(org_id="x", fiscal_year_start_month=13) # same 1..12 bound as Organization + OrgRecord(org_id="x", fiscal_year_start_month=13) # same 1..12 bound as Datasource def test_set_org_fields_mints_then_updates_only_passed_fields(tmp_path): diff --git a/tests/test_organization_context_e2e.py b/tests/test_organization_context_e2e.py index 74d848f7..0e575eec 100644 --- a/tests/test_organization_context_e2e.py +++ b/tests/test_organization_context_e2e.py @@ -28,21 +28,21 @@ from semantic_model import org_record as OR # noqa: E402 from semantic_model.cli import main as cli_main # noqa: E402 from semantic_model.models import ( # noqa: E402 + Datasource, DisplayConventions, - Organization, OrgRecord, SubjectArea, ) def _profile(root: Path, name: str, area: str, account_means: str) -> None: - org = Organization( + org = Datasource( datasource=name, subject_areas=[SubjectArea(name=area, description=f"{area} area")], ) build.write_tree(org, root / name) # key_terminology is written by the enrichment / set-terminology path, not write_tree — inject it so - # the source-specific vocabulary ("Account" resolves per source) round-trips through load_organization. + # the source-specific vocabulary ("Account" resolves per source) round-trips through load_datasource. org_yaml = root / name / "datasource.yaml" doc = yaml.safe_load(org_yaml.read_text()) doc["key_terminology"] = {"Account": account_means} diff --git a/tests/test_render_model_explorer.py b/tests/test_render_model_explorer.py index a28d4ed3..c1b144cd 100644 --- a/tests/test_render_model_explorer.py +++ b/tests/test_render_model_explorer.py @@ -127,7 +127,7 @@ def test_manifest_surfaces_full_model(profile_dir): "confidence": "confirmed", "signed_off_by": "agami_introspect", "signed_off_role": "system", "signed_off_at": "t"}]})) m = build_manifest(profile_dir, "test") - assert "MRR = monthly recurring revenue" in m["organization_md"] + assert "MRR = monthly recurring revenue" in m["datasource_md"] assert any(e["name"] == "customer" and e["maps_to"] == ["customers.id"] for e in m["entities"]) assert any(x["question"] == "how many orders" for x in m["examples"]) # the fixture's orders→customers FK relationship is surfaced diff --git a/tests/test_sample_database.py b/tests/test_sample_database.py index 8dd6b161..ffc19e65 100644 --- a/tests/test_sample_database.py +++ b/tests/test_sample_database.py @@ -82,7 +82,7 @@ def test_customer_names_are_unique(db): def test_full_name_is_not_sensitive(): """full_name is the display label for a customer — it must be queryable; only email/phone are sensitive.""" - org = L.load_organization(MODEL_DIR) + org = L.load_datasource(MODEL_DIR) cols = {c.name: c for area in org.subject_areas for t in area.tables_defined for c in t.columns if t.name == "customers"} assert cols["full_name"].sensitive is False assert cols["email"].sensitive is True @@ -90,7 +90,7 @@ def test_full_name_is_not_sensitive(): def test_model_validates(): - org = L.load_organization(MODEL_DIR) + org = L.load_datasource(MODEL_DIR) res = V.validate(org) assert res.ok, res.errors @@ -99,7 +99,7 @@ def test_context_surfaces_row_counts(): """The answer receipt's '≈N rows' provenance reads performance_hints.estimated_row_count from the assembled context. Regression guard for the 'rows unknown' bug, where the context/bundle include-list dropped performance_hints even though the model had it.""" - org = L.load_organization(MODEL_DIR) + org = L.load_datasource(MODEL_DIR) # the compound context fetch (default include) ctx = L.get_table_context(org, ["subscriptions", "plans"], area="agami-example") assert ctx["tables"]["subscriptions"]["performance_hints"]["estimated_row_count"] == 400 @@ -112,7 +112,7 @@ def test_context_surfaces_row_counts(): def test_revenue_metric_is_signed_off(): """The committed model ships signed-off metrics, so demo answers carry no 'not reviewed' warning.""" - org = L.load_organization(MODEL_DIR) + org = L.load_datasource(MODEL_DIR) metrics = {m.name: m for area in org.subject_areas for m in area.metrics} assert "revenue" in metrics assert metrics["revenue"].review_state == "approved" @@ -121,7 +121,7 @@ def test_revenue_metric_is_signed_off(): def test_fan_trap_is_refused(): """Summing the order-grain total across the line-item join double-counts — the pre-flight must refuse it. This is the headline demo.""" - org = L.load_organization(MODEL_DIR) + org = L.load_datasource(MODEL_DIR) sql = ( "SELECT cat.name, SUM(o.total_amount) FROM orders o " "JOIN order_items oi ON oi.order_id = o.id " @@ -134,7 +134,7 @@ def test_fan_trap_is_refused(): def test_chasm_trap_is_refused(): - org = L.load_organization(MODEL_DIR) + org = L.load_datasource(MODEL_DIR) sql = ( "SELECT c.id, SUM(o.total_amount), SUM(s.id) FROM customers c " "JOIN orders o ON o.customer_id = c.id " @@ -206,7 +206,7 @@ def test_correct_revenue_by_category(db): def test_sensitive_projection_refuses_raw(): """Projecting a raw sensitive value (bare, aliased, via *, or via MIN/MAX) is refused.""" - org = L.load_organization(MODEL_DIR) + org = L.load_datasource(MODEL_DIR) for sql in [ "SELECT email FROM customers", "SELECT * FROM customers", @@ -219,7 +219,7 @@ def test_sensitive_projection_refuses_raw(): def test_sensitive_projection_allows_aggregate_filter_and_nonsensitive(): """COUNT/COUNT(DISTINCT), WHERE-only use, and non-sensitive columns are allowed.""" - org = L.load_organization(MODEL_DIR) + org = L.load_datasource(MODEL_DIR) for sql in [ "SELECT COUNT(DISTINCT email) FROM customers", "SELECT COUNT(email) FROM customers", @@ -234,7 +234,7 @@ def test_sensitive_projection_refuses_set_operation_arm(): """A sensitive column projected in ANY set-operation arm is refused. A UNION parses to exp.SetOperation (not exp.Select), so a gate that only inspects the root SELECT would let `SELECT id … UNION SELECT email …` slip PII straight past the gate.""" - org = L.load_organization(MODEL_DIR) + org = L.load_datasource(MODEL_DIR) for sql in [ "SELECT id FROM customers UNION SELECT email FROM customers", "SELECT full_name FROM customers UNION ALL SELECT email FROM customers", @@ -247,7 +247,7 @@ def test_sensitive_projection_refuses_set_operation_arm(): def test_sensitive_projection_allows_union_of_nonsensitive(): """A set operation whose every arm projects only non-sensitive columns still passes — the arm-walking fix must not over-refuse a clean UNION.""" - org = L.load_organization(MODEL_DIR) + org = L.load_datasource(MODEL_DIR) sql = "SELECT id FROM customers UNION SELECT country FROM customers" assert RT.check_sensitive_projection(sql, org).action == "allow", sql diff --git a/tests/test_second_order_metrics.py b/tests/test_second_order_metrics.py index 21fc1b96..30a49fde 100644 --- a/tests/test_second_order_metrics.py +++ b/tests/test_second_order_metrics.py @@ -31,7 +31,7 @@ def _metric(name, binding, *, base=None, inner_grain=None, tables=("orders",)): def _org(metrics): sa = m.SubjectArea(name="sales", description="d", metrics=metrics) - return m.Organization(datasource="o", version=1, subject_areas=[sa]) + return m.Datasource(datasource="o", version=1, subject_areas=[sa]) def test_synthesizes_avg_daily_revenue_cte(): diff --git a/tests/test_semantic_model_cli.py b/tests/test_semantic_model_cli.py index 251891db..7d0c59f3 100644 --- a/tests/test_semantic_model_cli.py +++ b/tests/test_semantic_model_cli.py @@ -168,13 +168,13 @@ def test_review_item_matches_dashboard_contract(tmp_path): # carry them, else the card shows no description (bug 2) and the feedback generator # omits `by role=` for sign-off (bug 3, which buckets by rule_1). from semantic_model import curate - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) curate.write_items(tmp_path, "s", "metric", [ {"name": "revenue", "calculation": "Total revenue (USD)", "bindings": {"PostgreSQL": "SUM(orders.total)"}, "source_tables": ["orders"], "confidence": "inferred", "review_state": "unreviewed"}]) - org = load_organization(tmp_path, include_rejected=True) + org = load_datasource(tmp_path, include_rejected=True) m = next(it for it in curate.all_items(org, scope="all") if it["entity_type"] == "metric") assert m["rule_1"] is True assert any(s["text"] == "Total revenue (USD)" for s in m["signals"]) @@ -188,7 +188,7 @@ def test_review_items_scope_preseed_covers_metrics_and_entities_not_joins(tmp_pa # the "curate before examples" gate: seeds depend on metrics + entities, so preseed # includes both — but NOT relationships (those stay lazy / auto-approved FKs) from semantic_model import curate - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) # has a system-approved relationship (auto, FK) curate.write_items(tmp_path, "s", "metric", [ {"name": "rev", "calculation": "sum", "bindings": {"PostgreSQL": "SUM(orders.total)"}, @@ -196,7 +196,7 @@ def test_review_items_scope_preseed_covers_metrics_and_entities_not_joins(tmp_pa curate.write_items(tmp_path, "s", "entity", [ {"name": "order", "plural": "orders", "maps_to": [{"table": "orders", "column": "id", "primary": True}], "confidence": "inferred", "review_state": "unreviewed"}]) - org = load_organization(tmp_path, include_rejected=True) + org = load_datasource(tmp_path, include_rejected=True) preseed = curate.all_items(org, scope="preseed") types = {it["entity_type"] for it in preseed} assert "metric" in types and "entity" in types @@ -208,12 +208,12 @@ def test_review_items_scope_rule1_returns_only_signoff_items(tmp_path): # the Phase 4 gate uses --scope rule1 so the rendered count == the sign-off count; # no skill-side hand-filtering, no env var. rule1 = metrics/named-filters in review tab. from semantic_model import curate - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) # has a system-approved relationship (tab=auto, not in review) curate.write_items(tmp_path, "s", "metric", [ {"name": "Revenue", "calculation": "sum", "bindings": {"PostgreSQL": "SUM(orders.total)"}, "source_tables": ["orders"], "confidence": "inferred", "review_state": "unreviewed"}]) - org = load_organization(tmp_path, include_rejected=True) + org = load_datasource(tmp_path, include_rejected=True) rule1 = curate.all_items(org, scope="rule1") assert rule1 and all(it["rule"] == 1 and it["tab"] == "review" for it in rule1) assert any(it["entity_type"] == "metric" for it in rule1) @@ -244,7 +244,7 @@ def test_curate_edit_op_sets_enrichment_fields(tmp_path): # enrichment edits (descriptions / caveats / default_filters / value_transform) go # through `sm curate` edit ops — not a hand-edited or scripted table YAML from semantic_model import curate - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) res = curate.apply(tmp_path, [ {"op": "edit", "kind": "table", "area": "s", "name": "orders", @@ -256,7 +256,7 @@ def test_curate_edit_op_sets_enrichment_fields(tmp_path): {"op": "edit", "kind": "table", "area": "s", "name": "orders", "column": "total", "field": "value_transform", "value": "ABS(total)"}]) assert res.validated and len(res.applied) == 4 - t = load_organization(tmp_path).subject_areas[0].defined_table("orders") + t = load_datasource(tmp_path).subject_areas[0].defined_table("orders") assert t.caveats == ["Excludes test orders."] assert t.default_filters == ["{alias}.deleted_at IS NOT NULL"] assert t.description == "One row per order." @@ -267,7 +267,7 @@ def test_curate_edit_op_sets_structured_fields(tmp_path): # the model-explorer's structured editors emit list/object edit-ops; curate applies # them (caveats list, entity maps_to, relationship cardinality) from semantic_model import curate - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) # _model has an entity? no — add one + use the existing rel/columns (tmp_path / "subject_areas" / "s" / "entities").mkdir(parents=True, exist_ok=True) @@ -283,7 +283,7 @@ def test_curate_edit_op_sets_structured_fields(tmp_path): {"op": "edit", "kind": "relationship", "area": "s", "name": "order_items->orders", "field": "relationship", "value": "one_to_many"}]) assert res.validated and len(res.applied) == 3 - sa = load_organization(tmp_path).subject_areas[0] + sa = load_datasource(tmp_path).subject_areas[0] assert sa.defined_table("orders").get_column("total").caveats == ["excludes refunds", "net of tax"] assert [(m.table, m.column) for m in next(e for e in sa.entities if e.name == "buyer").maps_to] == [("order_items", "order_id")] assert next(r for r in sa.relationships if r.from_table == "order_items").relationship == "one_to_many" @@ -381,9 +381,9 @@ def test_coverage_flags_tables_enrichment_skipped(tmp_path): # the enrichment-completeness check: a freshly-introspected model has 0 column # descriptions, so every table reads as "enrichment never ran" → ok:false. from semantic_model import curate - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) - cov = curate.column_coverage(load_organization(tmp_path, include_rejected=True)) + cov = curate.column_coverage(load_datasource(tmp_path, include_rejected=True)) assert cov["ok"] is False assert set(cov["unenriched_tables"]) == {"orders", "order_items"} assert cov["totals"]["described"] == 0 @@ -393,7 +393,7 @@ def test_coverage_ok_when_each_table_has_some_descriptions(tmp_path): # ok flips true once each table has >=1 described/ai_unknown column — self-evident # columns (id, order_id) legitimately stay blank and do NOT hold the gate. from semantic_model import curate - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) curate.apply(tmp_path, [ {"op": "edit", "kind": "table", "area": "s", "name": "orders", "column": "total", @@ -402,7 +402,7 @@ def test_coverage_ok_when_each_table_has_some_descriptions(tmp_path): "field": "description_source", "value": "ai_unknown"}, {"op": "edit", "kind": "table", "area": "s", "name": "order_items", "column": "qty", "field": "description", "value": "quantity ordered"}]) - cov = curate.column_coverage(load_organization(tmp_path, include_rejected=True)) + cov = curate.column_coverage(load_datasource(tmp_path, include_rejected=True)) assert cov["ok"] is True and cov["unenriched_tables"] == [] assert cov["totals"]["described"] == 2 and cov["totals"]["ai_unknown"] == 1 @@ -614,9 +614,9 @@ def test_apply_reverts_writes_without_git_on_validation_failure(tmp_path): # apply()'s revert must NOT depend on a git repo (the artifacts dir usually isn't one). # A batch whose later op makes the model invalid rolls back the earlier valid write too. from semantic_model import curate - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) # not a git repo - before = load_organization(tmp_path).subject_areas[0].defined_table("orders").description + before = load_datasource(tmp_path).subject_areas[0].defined_table("orders").description res = curate.apply(tmp_path, [ {"op": "edit", "kind": "table", "area": "s", "name": "orders", "field": "description", "value": "CHANGED"}, # valid, writes @@ -624,7 +624,7 @@ def test_apply_reverts_writes_without_git_on_validation_failure(tmp_path): "column": "total", "field": "type", "value": "not_a_type"}, # makes the model unloadable ]) assert not res.validated and res.applied == [] - after = load_organization(tmp_path).subject_areas[0].defined_table("orders").description + after = load_datasource(tmp_path).subject_areas[0].defined_table("orders").description assert after == before == "o" # the first write was rolled back despite no git @@ -633,7 +633,7 @@ def test_column_groups_edit_reconciles_stale_expose(tmp_path): # named the OLD groups must be reconciled, else the model fails validation. import yaml as y from semantic_model import curate - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) tp = tmp_path / "subject_areas" / "s" / "tables" / "orders.yaml" td = y.safe_load(tp.read_text()) @@ -652,20 +652,20 @@ def test_column_groups_edit_reconciles_stale_expose(tmp_path): assert res.validated and res.applied tr = next(t for t in y.safe_load(sap.read_text())["tables"] if t["table"] == "orders") assert not tr.get("expose_column_groups") # stale exposes reconciled away (new set covers all) - assert set(load_organization(tmp_path).subject_areas[0].defined_table("orders").column_groups) == {"identity", "rest"} + assert set(load_datasource(tmp_path).subject_areas[0].defined_table("orders").column_groups) == {"identity", "rest"} def test_set_terminology_writes_glossary_to_org_yaml(tmp_path): # the packaged path for the decoded-abbreviation legend: writes datasource.yaml key_terminology, # validates, merges over existing terms (so a re-run doesn't clobber a human's edits). - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) terms = tmp_path / "terms.json" terms.write_text(json.dumps({"MRR": "monthly recurring revenue", "ARR": "annual recurring revenue"})) rc, out = _run(["set-terminology", str(tmp_path), "--file", str(terms)]) d = json.loads(out) assert rc == 0 and d["validated"] and d["applied"] - assert load_organization(tmp_path).key_terminology == { + assert load_datasource(tmp_path).key_terminology == { "MRR": "monthly recurring revenue", "ARR": "annual recurring revenue"} @@ -673,13 +673,13 @@ def test_curate_edit_sets_semantic_column_groups(tmp_path): # the column-group refinement write path: enrichment overwrites the engine's prefix # buckets with named semantic groups via a normal curate edit op. from semantic_model import curate - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _model(tmp_path) res = curate.apply(tmp_path, [{"op": "edit", "kind": "table", "area": "s", "name": "orders", "field": "column_groups", "value": {"identity": ["id"], "lifecycle": ["deleted_at"], "money": ["total"]}}]) assert res.validated and res.applied - t = load_organization(tmp_path).subject_areas[0].defined_table("orders") + t = load_datasource(tmp_path).subject_areas[0].defined_table("orders") assert t.column_groups == {"identity": ["id"], "lifecycle": ["deleted_at"], "money": ["total"]} diff --git a/tests/test_semantic_model_curate.py b/tests/test_semantic_model_curate.py index 52dc1e40..22b39335 100644 --- a/tests/test_semantic_model_curate.py +++ b/tests/test_semantic_model_curate.py @@ -73,7 +73,7 @@ def _write_model(root: Path, *, git: bool = True) -> None: def test_review_queue_partitions_rule1_rule2(tmp_path): _write_model(tmp_path, git=False) - org = loader.load_organization(tmp_path) + org = loader.load_datasource(tmp_path) q = curate.review_queue(org) assert q["counts"]["rule_1"] == 1 # the proposed metric assert q["counts"]["rule_2"] == 1 # the inferred relationship @@ -83,7 +83,7 @@ def test_review_queue_partitions_rule1_rule2(tmp_path): def test_model_tree_shows_columns_and_state(tmp_path): _write_model(tmp_path, git=False) - org = loader.load_organization(tmp_path, include_rejected=True) + org = loader.load_datasource(tmp_path, include_rejected=True) tree = curate.model_tree(org) orders = next(t for t in tree["subject_areas"][0]["tables"] if t["table"] == "orders") assert orders["review_state"] == "approved" @@ -96,10 +96,10 @@ def test_exclude_table_hides_from_runtime(tmp_path): res = curate.apply(tmp_path, [{"op": "exclude", "kind": "table", "area": "sales", "name": "orders"}]) assert res.validated and res.applied - runtime = loader.load_organization(tmp_path) # drops rejected + runtime = loader.load_datasource(tmp_path) # drops rejected assert not any(t.name == "orders" for sa in runtime.subject_areas for t in sa.tables_defined) # still visible with include_rejected (so the explorer can toggle it back) - full = loader.load_organization(tmp_path, include_rejected=True) + full = loader.load_datasource(tmp_path, include_rejected=True) assert any(t.name == "orders" for sa in full.subject_areas for t in sa.tables_defined) @@ -107,7 +107,7 @@ def test_include_restores(tmp_path): _write_model(tmp_path) curate.apply(tmp_path, [{"op": "exclude", "kind": "table", "area": "sales", "name": "orders"}]) curate.apply(tmp_path, [{"op": "include", "kind": "table", "area": "sales", "name": "orders"}]) - runtime = loader.load_organization(tmp_path) + runtime = loader.load_datasource(tmp_path) assert any(t.name == "orders" for sa in runtime.subject_areas for t in sa.tables_defined) @@ -116,7 +116,7 @@ def test_exclude_column(tmp_path): res = curate.apply(tmp_path, [{"op": "exclude", "kind": "table", "area": "sales", "name": "orders", "column": "ssn"}]) assert res.validated - orders = loader.load_organization(tmp_path).subject_areas[0].defined_table("orders") + orders = loader.load_datasource(tmp_path).subject_areas[0].defined_table("orders") assert not any(c.name == "ssn" for c in orders.columns) @@ -126,7 +126,7 @@ def test_approve_metric_records_signoff(tmp_path): "name": "order_count", "at": "2026-06-09T00:00:00Z"}], signer="reviewer@example.com", role="cto") assert res.validated - org = loader.load_organization(tmp_path) + org = loader.load_datasource(tmp_path) mm = org.subject_areas[0].metrics[0] assert mm.review_state == "approved" and mm.signed_off_by == "reviewer@example.com" # no longer in the Rule 1 queue @@ -139,7 +139,7 @@ def test_approve_relationship(tmp_path): "name": "orders->customers", "at": "2026-06-09T00:00:00Z"}], signer="reviewer@example.com", role="cto") assert res.validated - rel = loader.load_organization(tmp_path).subject_areas[0].relationships[0] + rel = loader.load_datasource(tmp_path).subject_areas[0].relationships[0] assert rel.review_state == "approved" and rel.signed_off_by == "reviewer@example.com" @@ -155,7 +155,7 @@ def test_edit_relationship_on_clause(tmp_path): "field": "to_column", "value": None}, ]) assert res.validated, res.errors - rel = loader.load_organization(tmp_path).subject_areas[0].relationships[0] + rel = loader.load_datasource(tmp_path).subject_areas[0].relationships[0] assert rel.on and "CAST" in rel.on @@ -174,7 +174,7 @@ def test_approve_resolves_slugged_metric_name(tmp_path): "name": "total event sales", "at": "2026-06-12T00:00:00Z"}], signer="x@y.com", role="cto") assert res.applied and not res.skipped, res.skipped - m2 = next(x for x in loader.load_organization(tmp_path).subject_areas[0].metrics + m2 = next(x for x in loader.load_datasource(tmp_path).subject_areas[0].metrics if x.name == "total event sales") assert m2.review_state == "approved" @@ -191,7 +191,7 @@ def test_edit_op_auto_resolves_area_when_omitted(tmp_path): res = curate.apply(tmp_path, [{"op": "edit", "kind": "table", "name": "orders", "field": "description", "value": "all orders", "source": "ai"}]) assert res.validated and res.applied and not res.skipped, res.skipped - orders = loader.load_organization(tmp_path).subject_areas[0].defined_table("orders") + orders = loader.load_datasource(tmp_path).subject_areas[0].defined_table("orders") assert orders.description == "all orders" assert orders.description_source == "ai_unvalidated" # source:"ai" stamped diff --git a/tests/test_semantic_model_introspect.py b/tests/test_semantic_model_introspect.py index eba2a332..7e5ebf3e 100644 --- a/tests/test_semantic_model_introspect.py +++ b/tests/test_semantic_model_introspect.py @@ -242,7 +242,7 @@ def test_introspect_writes_canonical_tree_and_loads_back(tmp_path): artifacts_dir=tmp_path, dry_run=False) root = tmp_path / "shop" assert (root / "datasource.yaml").exists() - reloaded = L.load_organization(root) + reloaded = L.load_datasource(root) assert V.validate(reloaded).ok assert {t.name for sa in reloaded.subject_areas for t in sa.tables_defined} == {"customers", "orders"} @@ -651,7 +651,7 @@ def test_same_named_tables_in_two_schemas_both_survive(tmp_path): assert (root / "subject_areas" / "crm" / "tables" / "products.yaml").exists() # reload from disk -> still 4 tables (write+read round-trips without loss) from semantic_model import loader as L - reloaded = L.load_organization(root) + reloaded = L.load_datasource(root) assert sum(len(sa.tables_defined) for sa in reloaded.subject_areas) == 4 # the probed join binds within billing (same-schema), not across to crm.products bil = next(sa for sa in reloaded.subject_areas if sa.name == "billing") @@ -821,7 +821,7 @@ def test_introspect_append_merges_batches(tmp_path): I.introspect("shop", "postgres", runner=_append_runner, artifacts_dir=tmp_path, tables=["public.order_items"], append=True) - org = L.load_organization(tmp_path / "shop") + org = L.load_datasource(tmp_path / "shop") tnames = {t.name for sa in org.subject_areas for t in sa.tables_defined} assert tnames == {"orders", "customers", "order_items"} # union — nothing lost, no re-query allrels = [r for sa in org.subject_areas for r in sa.relationships] + list(org.cross_subject_area_relationships) @@ -837,6 +837,6 @@ def test_introspect_append_relisting_table_no_duplicate(tmp_path): # batch 2 re-lists orders (already built) + adds order_items I.introspect("shop", "postgres", runner=_append_runner, artifacts_dir=tmp_path, tables=["public.orders", "public.order_items"], append=True) - org = L.load_organization(tmp_path / "shop") + org = L.load_datasource(tmp_path / "shop") names = [t.name for sa in org.subject_areas for t in sa.tables_defined] assert sorted(names) == ["customers", "order_items", "orders"] # each table exactly once diff --git a/tests/test_semantic_model_loader.py b/tests/test_semantic_model_loader.py index 12e8c97b..077181a8 100644 --- a/tests/test_semantic_model_loader.py +++ b/tests/test_semantic_model_loader.py @@ -31,7 +31,7 @@ def _wide_org(): tables=[m.TableRef(storage_connection="c", schema="analytics", table="wide", expose_column_groups=["metrics", "location"])], tables_defined=[t]) - return m.Organization(datasource="Acme", + return m.Datasource(datasource="Acme", storage_connections=[m.StorageConnection(name="c", storage_type="PostgreSQL")], subject_areas=[sa]) @@ -106,7 +106,7 @@ def test_disk_round_trip(tmp_path): (root / "subject_areas" / "area" / "relationships.yaml").write_text(yaml.safe_dump({ "relationships": [], })) - org = L.load_organization(root) + org = L.load_datasource(root) assert org.datasource == "O" sa = org.subject_area("area") assert sa is not None and sa.defined_table("t").grain == ["id"] diff --git a/tests/test_semantic_model_models.py b/tests/test_semantic_model_models.py index c745d692..ab2bc64f 100644 --- a/tests/test_semantic_model_models.py +++ b/tests/test_semantic_model_models.py @@ -160,13 +160,13 @@ def test_cross_relationship_requires_areas(): assert ok.from_subject_area == "A" -# --- Organization --- +# --- Datasource --- def test_org_happy_and_accessors(): t = m.Table(name="t", schema="s", storage_connection="c", grain=["id"], description="d", columns=[_col("id")]) - org = m.Organization(datasource="O", + org = m.Datasource(datasource="O", storage_connections=[m.StorageConnection(name="c", storage_type="PostgreSQL")], subject_areas=[m.SubjectArea(name="sa", tables_defined=[t])]) assert org.subject_area("sa").defined_table("t") is t @@ -175,4 +175,4 @@ def test_org_happy_and_accessors(): def test_org_bad_fiscal_month_rejected(): with pytest.raises(ValidationError): - m.Organization(datasource="O", fiscal_year_start_month=13) + m.Datasource(datasource="O", fiscal_year_start_month=13) diff --git a/tests/test_semantic_model_runtime.py b/tests/test_semantic_model_runtime.py index fb76daa3..4b211f7a 100644 --- a/tests/test_semantic_model_runtime.py +++ b/tests/test_semantic_model_runtime.py @@ -26,7 +26,7 @@ def _sales_org(): m.Relationship(from_table="tickets", to_table="customers", from_column="customer_id", to_column="id", relationship="many_to_one"), ] - return m.Organization(datasource="Shop", + return m.Datasource(datasource="Shop", subject_areas=[m.SubjectArea(name="sales", relationships=rels)]) @@ -120,7 +120,7 @@ def _entity_org(): maps_to=[m.EntityMapping(table="orders", column="order_no", primary=True)]) o2 = m.Entity(name="Shipment", value_pattern=r"^SH\w+$", maps_to=[m.EntityMapping(table="shipments", column="ship_no", primary=True)]) - return m.Organization(datasource="Acme", + return m.Datasource(datasource="Acme", subject_areas=[m.SubjectArea(name="b", entities=[o1, o2])]) @@ -168,7 +168,7 @@ def _filter_org(): m.Column(name="total", type="decimal"), m.Column(name="tenant_id", type="integer")], default_filters=["{alias}.deleted_at IS NULL"]) - return m.Organization(datasource="S", + return m.Datasource(datasource="S", subject_areas=[m.SubjectArea(name="s", tables=[m.TableRef(storage_connection="c", schema="public", table="orders")], tables_defined=[t])]) diff --git a/tests/test_semantic_model_validator_v2.py b/tests/test_semantic_model_validator_v2.py index 1a145cb9..e3791fa2 100644 --- a/tests/test_semantic_model_validator_v2.py +++ b/tests/test_semantic_model_validator_v2.py @@ -1,6 +1,6 @@ """Unit tests for semantic_model/validator.py — one happy + one failure per rule. -Each test builds a minimal Organization that isolates a single rule. +Each test builds a minimal Datasource that isolates a single rule. """ from __future__ import annotations @@ -25,7 +25,7 @@ def _conn(name="c"): def _org(area, conn="c", **kw): - return m.Organization(datasource="O", storage_connections=[_conn(conn)], + return m.Datasource(datasource="O", storage_connections=[_conn(conn)], subject_areas=[area], **kw) @@ -84,7 +84,7 @@ def test_table_ref_resolves_org_wide(): tables_defined=[t]) b = m.SubjectArea(name="B", tables=[m.TableRef(storage_connection="c", schema="s", table="shared")], tables_defined=[]) - org = m.Organization(datasource="O", storage_connections=[_conn()], subject_areas=[a, b]) + org = m.Datasource(datasource="O", storage_connections=[_conn()], subject_areas=[a, b]) res = v.validate(org) assert "orphan_table_ref" not in _codes(res) @@ -284,7 +284,7 @@ def test_cross_area_entity_collision_warns(): tables_defined=[rr], entities=[m.Entity(name="Account", maps_to=[m.EntityMapping(table="rev", column="id", primary=True)])]) - org = m.Organization(datasource="O", storage_connections=[_conn()], + org = m.Datasource(datasource="O", storage_connections=[_conn()], subject_areas=[area_a, area_b]) res = v.validate(org) assert "cross_area_entity_collision" in _codes(res) @@ -308,7 +308,7 @@ def test_executable_mismatch_on_cross_edge(): edge = m.CrossSubjectAreaRelationship(from_table="a", to_table="b", from_column="x", to_column="y", relationship="many_to_one", executable="same_engine", from_subject_area="A", to_subject_area="B") - org = m.Organization(datasource="O", + org = m.Datasource(datasource="O", storage_connections=[_conn("c1"), _conn("c2")], subject_areas=[area_a, area_b], cross_subject_area_relationships=[edge]) diff --git a/tests/test_sensitive_gate.py b/tests/test_sensitive_gate.py index 511d6dcc..8ec3695d 100644 --- a/tests/test_sensitive_gate.py +++ b/tests/test_sensitive_gate.py @@ -20,7 +20,7 @@ def _org(cols): t = m.Table(name="users", schema="public", storage_connection="c", grain=["id"], columns=cols) sa = m.SubjectArea(name="area", description="d", tables_defined=[t]) - return m.Organization(datasource="o", version=1, subject_areas=[sa]) + return m.Datasource(datasource="o", version=1, subject_areas=[sa]) def test_counts_flagged_pii(): @@ -50,7 +50,7 @@ def test_sensitive_columns_under_excluded_table_not_counted(): columns=[m.Column(name="email", type="string", sensitive=True), m.Column(name="ssn", type="string", sensitive=True)]) sa = m.SubjectArea(name="area", description="d", tables_defined=[t]) - org = m.Organization(datasource="o", version=1, subject_areas=[sa]) + org = m.Datasource(datasource="o", version=1, subject_areas=[sa]) assert C.sensitive_columns(org)["count"] == 0 diff --git a/tests/test_table_scope_gate.py b/tests/test_table_scope_gate.py index 8f7ea8ee..faf77704 100644 --- a/tests/test_table_scope_gate.py +++ b/tests/test_table_scope_gate.py @@ -31,7 +31,7 @@ def _scope_org(): def _t(name): return m.Table(name=name, schema="public", storage_connection="c", grain=["id"], description=name, columns=[m.Column(name="id", type="integer")]) - return m.Organization(datasource="Shop", + return m.Datasource(datasource="Shop", subject_areas=[m.SubjectArea(name="sales", tables_defined=[_t("orders"), _t("customers")])]) @@ -89,7 +89,7 @@ def test_case_insensitive_match(): def test_empty_model_allows(): - org = m.Organization(datasource="Empty", subject_areas=[m.SubjectArea(name="s")]) + org = m.Datasource(datasource="Empty", subject_areas=[m.SubjectArea(name="s")]) assert rt.check_table_scope("SELECT * FROM anything", org).action == "allow" diff --git a/tests/test_units.py b/tests/test_units.py index 664e10c1..5bab2389 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -95,9 +95,9 @@ def test_resolve_result_units_traces_aggregates(tmp_path): # under an alias; COUNT and ratios do not become currency pytest.importorskip("sqlglot") from semantic_model import runtime as RT - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _currency_model(tmp_path) - org = load_organization(tmp_path) + org = load_datasource(tmp_path) # named results carry the unit by name (+ a positional #0 key alongside) assert RT.resolve_result_units(org, "SELECT SUM(amount) AS total FROM loans")["total"] == "INR" assert RT.resolve_result_units(org, "SELECT AVG(amount) AS a FROM loans")["a"] == "INR" @@ -115,7 +115,7 @@ def test_resolve_result_units_dimensional_ratios(tmp_path): pytest.importorskip("sqlglot") import yaml from semantic_model import runtime as RT - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource _currency_model(tmp_path) # add a second currency col + a percent col t = tmp_path / "subject_areas" / "s" / "tables" / "loans.yaml" @@ -123,7 +123,7 @@ def test_resolve_result_units_dimensional_ratios(tmp_path): doc["columns"] += [{"name": "fee", "type": "decimal", "unit": "INR"}, {"name": "npa_pct", "type": "decimal", "unit": "percent"}] t.write_text(yaml.safe_dump(doc)) - org = load_organization(tmp_path) + org = load_datasource(tmp_path) R = lambda s: RT.resolve_result_units(org, s) assert R("SELECT SUM(amount)/COUNT(*) AS avg_ticket FROM loans")["avg_ticket"] == "INR" assert R("SELECT amount*1.18 AS with_tax FROM loans")["with_tax"] == "INR" @@ -157,7 +157,7 @@ def test_resolve_result_units_emits_date_format_token(tmp_path): pytest.importorskip("sqlglot") import yaml from semantic_model import runtime as RT - from semantic_model.loader import load_organization + from semantic_model.loader import load_datasource (tmp_path / "datasources" / "c").mkdir(parents=True) (tmp_path / "subject_areas" / "s" / "tables").mkdir(parents=True) (tmp_path / "datasource.yaml").write_text(yaml.safe_dump({ @@ -172,7 +172,7 @@ def test_resolve_result_units_emits_date_format_token(tmp_path): "name": "orders", "schema": "public", "storage_connection": "c", "grain": ["id"], "description": "o", "columns": [{"name": "id", "type": "integer", "primary_key": True}, {"name": "created_ts", "type": "integer", "date_format": "epoch_s", "timezone": "UTC"}]})) - org = load_organization(tmp_path) + org = load_datasource(tmp_path) assert RT.resolve_result_units(org, "SELECT created_ts FROM orders")["created_ts"] == "epoch_s" # propagates through MAX (the last timestamp is still that encoding) assert RT.resolve_result_units(org, "SELECT MAX(created_ts) AS last FROM orders")["last"] == "epoch_s" @@ -186,7 +186,7 @@ def test_format_table_applies_units_by_position(): def test_unit_round_trips_on_column_and_surfaces_in_context(tmp_path): import yaml - from semantic_model.loader import get_table_context, load_organization + from semantic_model.loader import get_table_context, load_datasource root = tmp_path (root / "datasources" / "c").mkdir(parents=True) (root / "subject_areas" / "s" / "tables").mkdir(parents=True) @@ -202,7 +202,7 @@ def test_unit_round_trips_on_column_and_surfaces_in_context(tmp_path): "name": "orders", "schema": "public", "storage_connection": "c", "grain": ["id"], "description": "o", "columns": [{"name": "id", "type": "integer", "primary_key": True}, {"name": "amount", "type": "decimal", "unit": "INR"}]})) - org = load_organization(root) + org = load_datasource(root) assert org.subject_areas[0].defined_table("orders").get_column("amount").unit == "INR" ctx = get_table_context(org, ["orders"], area="s") amount = next(c for c in ctx["tables"]["orders"]["columns"] if c["name"] == "amount") From bf47c922e459be0d2bc921f59ac4eddd2338784b Mon Sep 17 00:00:00 2001 From: vishal-agami Date: Sat, 25 Jul 2026 19:39:06 +0530 Subject: [PATCH 6/6] fix: address Copilot review of the symbol rename MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - dev/render_previews.py: the preview script (dev-only, outside tests + lint) still used the pre-rename shapes -> two crash-on-run bugs: * `_MODEL_DS` used the old top-level key `organization:` -> `Datasource.model_validate` failed (extra="forbid" + missing required `datasource`). Fixed the key. * `write_memory(..., organization=...)` -> the param is `datasource_doc`. Fixed the kwarg. (verified by actually running the script — it renders all previews now.) - Grammar: "an Datasource" -> "a Datasource" in 5 docstrings/comments (the Organization->Datasource sed left the wrong article). No product-path impact (render_previews is a manual UI-preview dev tool, not imported by the server or any flow). Gate green: ruff + 1557 tests + gitleaks + lib-drift. Co-Authored-By: Claude Opus 4.8 (1M context) --- dev/render_previews.py | 12 ++++++------ packages/agami-core/src/execute_sql.py | 2 +- packages/agami-core/src/model_store.py | 2 +- packages/agami-core/src/semantic_model/loader.py | 2 +- packages/agami-core/src/tools.py | 2 +- plugins/agami/lib/execute_sql.py | 2 +- tests/test_model_store_roundtrip.py | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dev/render_previews.py b/dev/render_previews.py index 4cde1549..92924f9b 100644 --- a/dev/render_previews.py +++ b/dev/render_previews.py @@ -127,8 +127,8 @@ def write(name: str, html: str) -> None: # tools read (load_datasource), so the previews can't drift from production. Neutral demo data only. from semantic_model.models import Datasource # noqa: E402 -_MODEL_ORG = { - "organization": "acme", +_MODEL_DS = { + "datasource": "acme", "version": 1, "description": "Acme Commerce — the deployed semantic model.", "fiscal_year_start_month": 1, @@ -248,16 +248,16 @@ def write(name: str, html: str) -> None: "from_subject_area": "Sales", "to_subject_area": "Catalog"}, ], } -_ORG_MD = ( +_DS_MD = ( "# About Acme Commerce\n\n" "Acme sells direct-to-consumer goods online. Every checkout writes an **orders** row.\n\n" "## Key terms\n\n" "- **Net revenue** excludes cancelled orders and nets refunds.\n" "- A **member** has a row in the loyalty table — not just anyone who checked out.\n" ) -model_store.write_datasource(_s, "SALES_DATA", Datasource.model_validate(_MODEL_ORG)) -model_store.write_datasource(_s, "MARKETING", Datasource.model_validate(_MODEL_ORG)) # → picker -model_store.write_memory(_s, "SALES_DATA", organization=_ORG_MD) +model_store.write_datasource(_s, "SALES_DATA", Datasource.model_validate(_MODEL_DS)) +model_store.write_datasource(_s, "MARKETING", Datasource.model_validate(_MODEL_DS)) # → picker +model_store.write_memory(_s, "SALES_DATA", datasource_doc=_DS_MD) model_store.write_model_version(_s, "SALES_DATA", "a1f4c39c0b2e", created_at="2026-06-27T09:00:00Z") _dss = model_store.list_datasources(_s) _org = model_store.load_datasource(_s, "SALES_DATA") diff --git a/packages/agami-core/src/execute_sql.py b/packages/agami-core/src/execute_sql.py index 865e3a9c..25a17525 100644 --- a/packages/agami-core/src/execute_sql.py +++ b/packages/agami-core/src/execute_sql.py @@ -867,7 +867,7 @@ def _hosted() -> bool: def _resolve_guard_model(profile: str): """Resolve the semantic model for the safety pass, mirroring `tools._load_org` (ACE-051): from the DB when one is configured (hosted — the `/artifacts` disk mount may be absent), else the - on-disk YAML (local). Returns an `Datasource` or None if neither is available. + on-disk YAML (local). Returns a `Datasource` or None if neither is available. The DB import is lazy AND env-guarded on purpose: the local executor runs from a stdlib-lean mirror that does not ship `store`/`model_store`, so we only reach for them when a DB is set. diff --git a/packages/agami-core/src/model_store.py b/packages/agami-core/src/model_store.py index 71a6a96a..66b33cfe 100644 --- a/packages/agami-core/src/model_store.py +++ b/packages/agami-core/src/model_store.py @@ -1,6 +1,6 @@ """Serve the semantic model from the DB — the read path + the deploy-time writer. -The model-loader seam is just "produce an `Datasource`": the file adapter is +The model-loader seam is just "produce a `Datasource`": the file adapter is `semantic_model.loader.load_datasource(root)`; this is the DB adapter, which rebuilds the **identical** `Datasource` from rows so every downstream tool (get_datasource_schema incl. sizing, the receipt) is untouched. YAML stays the source of truth — `write_datasource` seeds the diff --git a/packages/agami-core/src/semantic_model/loader.py b/packages/agami-core/src/semantic_model/loader.py index ec7c7bcc..9ecf8074 100644 --- a/packages/agami-core/src/semantic_model/loader.py +++ b/packages/agami-core/src/semantic_model/loader.py @@ -77,7 +77,7 @@ def _read_yaml(path: Path) -> Any: def load_datasource(root: str | Path, *, include_rejected: bool = False) -> Datasource: - """Parse a v2 profile directory into an Datasource model. + """Parse a v2 profile directory into a Datasource model. By default, entries the curator excluded (`review_state: rejected`) are dropped so the runtime never sees them. Pass `include_rejected=True` for the curation diff --git a/packages/agami-core/src/tools.py b/packages/agami-core/src/tools.py index b7eb5bde..1b09ed88 100644 --- a/packages/agami-core/src/tools.py +++ b/packages/agami-core/src/tools.py @@ -205,7 +205,7 @@ def check_read_only(sql: str) -> str | None: def _load_org(profile: str): - """Lazily load the semantic model for a profile, producing an `Datasource`. Two backends + """Lazily load the semantic model for a profile, producing a `Datasource`. Two backends behind one seam: when AGAMI_DB_URL is set the hosted server reads it from the DB; otherwise the local skill reads the YAML files (unchanged). Raises a clear error if the model deps (pydantic) aren't importable or there's no model for the profile.""" diff --git a/plugins/agami/lib/execute_sql.py b/plugins/agami/lib/execute_sql.py index 865e3a9c..25a17525 100644 --- a/plugins/agami/lib/execute_sql.py +++ b/plugins/agami/lib/execute_sql.py @@ -867,7 +867,7 @@ def _hosted() -> bool: def _resolve_guard_model(profile: str): """Resolve the semantic model for the safety pass, mirroring `tools._load_org` (ACE-051): from the DB when one is configured (hosted — the `/artifacts` disk mount may be absent), else the - on-disk YAML (local). Returns an `Datasource` or None if neither is available. + on-disk YAML (local). Returns a `Datasource` or None if neither is available. The DB import is lazy AND env-guarded on purpose: the local executor runs from a stdlib-lean mirror that does not ship `store`/`model_store`, so we only reach for them when a DB is set. diff --git a/tests/test_model_store_roundtrip.py b/tests/test_model_store_roundtrip.py index 0b60a61e..0122a0aa 100644 --- a/tests/test_model_store_roundtrip.py +++ b/tests/test_model_store_roundtrip.py @@ -1,6 +1,6 @@ """Serve the model from the DB — golden parity with the file loader (Slice C). -Two proofs: (1) writing then loading an Datasource through the DB is lossless for every object +Two proofs: (1) writing then loading a Datasource through the DB is lossless for every object type; (2) a model loaded from YAML files, seeded to the DB, and re-loaded from a *fresh* connection yields the identical Datasource — and tools._load_org / get_datasource_schema serve from the DB (files absent) when AGAMI_DB_URL is set.