diff --git a/Cargo.lock b/Cargo.lock index 5b9d7ed73..b0c3844d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6484,6 +6484,7 @@ dependencies = [ "criterion", "serde", "serde_json", + "sha2 0.10.9", "temper-runtime", "temper-spec", "thiserror 2.0.18", diff --git a/crates/temper-cli/src/migrate_turso_to_postgres.rs b/crates/temper-cli/src/migrate_turso_to_postgres.rs index cf616944d..7a1cf8c09 100644 --- a/crates/temper-cli/src/migrate_turso_to_postgres.rs +++ b/crates/temper-cli/src/migrate_turso_to_postgres.rs @@ -432,7 +432,10 @@ async fn migrate_specs( .filter(|hash| !hash.is_empty()) .unwrap_or_else(|| spec_content_hash(&row.ioa_source)); sqlx::query( - "INSERT INTO specs \ + "WITH cleared_staging AS ( \ + DELETE FROM staged_specs WHERE tenant = $1 AND entity_type = $2 \ + ) \ + INSERT INTO specs \ (tenant, entity_type, ioa_source, csdl_xml, version, verified, verification_status, \ levels_passed, levels_total, verification_result, content_hash, committed, updated_at) \ VALUES ($1, $2, $3, $4, 1, $5, $6, $7, $8, $9, $10, $11, $12) \ @@ -2077,6 +2080,23 @@ mod tests { .await .expect("put source blob"); + let pool = PgPool::connect(&database_url).await.expect("target pool"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("target migrations"); + sqlx::query( + "INSERT INTO staged_specs \ + (tenant, entity_type, ioa_source, csdl_xml, content_hash, version, updated_at) \ + VALUES ($1, 'SmokeEntity', 'stale staged bytes', '', 'stale-hash', 1, now()) \ + ON CONFLICT (tenant, entity_type) DO UPDATE SET \ + ioa_source = EXCLUDED.ioa_source, csdl_xml = EXCLUDED.csdl_xml, \ + content_hash = EXCLUDED.content_hash, updated_at = now()", + ) + .bind(&tenant) + .execute(&pool) + .await + .expect("seed stale target staging"); + run(MigrationOptions { tenant: tenant.clone(), dry_run: false, @@ -2090,7 +2110,15 @@ mod tests { .await .expect("run migration"); - let pool = PgPool::connect(&database_url).await.expect("target pool"); + let stale_staged_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*)::bigint FROM staged_specs \ + WHERE tenant = $1 AND entity_type = 'SmokeEntity'", + ) + .bind(&tenant) + .fetch_one(&pool) + .await + .expect("count stale staging"); + assert_eq!(stale_staged_count, 0); let event_count: i64 = sqlx::query_scalar("SELECT COUNT(*)::bigint FROM events WHERE tenant = $1") .bind(&tenant) diff --git a/crates/temper-cli/src/serve/bootstrap.rs b/crates/temper-cli/src/serve/bootstrap.rs index 2e206e453..bdc98de2e 100644 --- a/crates/temper-cli/src/serve/bootstrap.rs +++ b/crates/temper-cli/src/serve/bootstrap.rs @@ -212,6 +212,18 @@ pub(super) fn load_webhooks(apps: &[(String, String)]) -> Option std::collections::BTreeSet { + state + .server + .registry + .read() + .map(|registry| registry.tenant_ids().into_iter().cloned().collect()) + .unwrap_or_else(|error| { + eprintln!(" Warning: registry lock poisoned during hydration: {error}"); + std::collections::BTreeSet::new() + }) +} + pub(super) async fn hydrate_entities(state: &PlatformState, apps: &[(String, String)]) { if state.server.storage_stack.is_none() { return; @@ -225,15 +237,9 @@ pub(super) async fn hydrate_entities(state: &PlatformState, apps: &[(String, Str ) }) .unwrap_or(false); - let mut all_tenants = Vec::new(); + let mut all_tenants = registered_hydration_tenants(state); for (tenant, _dir) in apps { - let tenant_id = TenantId::new(tenant.as_str()); - if eager_hydrate { - state.server.hydrate_from_store(&tenant_id).await; - } else { - state.server.populate_index_from_store(&tenant_id).await; - } - all_tenants.push(tenant_id); + all_tenants.insert(TenantId::new(tenant.as_str())); } // In TenantRouted mode, also hydrate all registered tenants. if let Some(provider) = state @@ -243,13 +249,14 @@ pub(super) async fn hydrate_entities(state: &PlatformState, apps: &[(String, Str .and_then(|stack| stack.turso.clone()) { for tenant in provider.connected_tenants().await { - let tenant_id = TenantId::new(&tenant); - if eager_hydrate { - state.server.hydrate_from_store(&tenant_id).await; - } else { - state.server.populate_index_from_store(&tenant_id).await; - } - all_tenants.push(tenant_id); + all_tenants.insert(TenantId::new(&tenant)); + } + } + for tenant_id in &all_tenants { + if eager_hydrate { + state.server.hydrate_from_store(tenant_id).await; + } else { + state.server.populate_index_from_store(tenant_id).await; } } @@ -406,16 +413,17 @@ pub(super) async fn recover_secrets(state: &PlatformState) { } } -/// Load the verification cache from Turso for a tenant (hash + verified status). +/// Load the verification cache from the active platform store for a tenant. /// -/// Routes to the per-tenant store in TenantRouted mode. -/// Returns an empty map if no Turso store is available. +/// Routes to a per-tenant Turso store in TenantRouted mode and to the shared +/// tenant-scoped Postgres store otherwise. Returns an empty map when platform +/// persistence is unavailable. async fn load_verified_cache( state: &PlatformState, tenant: &str, ) -> std::collections::BTreeMap { - if let Some(turso) = state.server.turso_store_for_tenant(tenant).await { - match turso.load_verification_cache(tenant).await { + if let Some(store) = state.server.platform_store_for_tenant(tenant).await { + match store.load_verification_cache(tenant).await { Ok(cache) => cache, Err(e) => { eprintln!(" Warning: failed to load verification cache for {tenant}: {e}"); @@ -435,16 +443,20 @@ async fn load_verified_cache( pub(super) async fn bootstrap_tenants(state: &PlatformState, apps: &[(String, String)]) { let sys_cache = load_verified_cache(state, "temper-system").await; let sys_hashes = temper_platform::bootstrap_system_tenant(state, &sys_cache); - if let Some(turso) = state.server.turso_store_for_tenant("temper-system").await { - temper_platform::persist_system_verification(&turso, &sys_hashes, &sys_cache).await; + if let Some(store) = state + .server + .platform_store_for_tenant("temper-system") + .await + { + temper_platform::persist_system_verification(store.as_ref(), &sys_hashes, &sys_cache).await; } let default_cache = load_verified_cache(state, "default").await; let default_hashes = temper_platform::bootstrap_agent_specs(state, "default", false, &default_cache); - if let Some(turso) = state.server.turso_store_for_tenant("default").await { + if let Some(store) = state.server.platform_store_for_tenant("default").await { temper_platform::persist_agent_verification( - &turso, + store.as_ref(), "default", &default_hashes, &default_cache, @@ -457,8 +469,9 @@ pub(super) async fn bootstrap_tenants(state: &PlatformState, apps: &[(String, St // App tenants already have user specs loaded in Phase 2; merge the // built-in agent OS entities so we do not replace their entity-set map. let hashes = temper_platform::bootstrap_agent_specs(state, tenant, true, &cache); - if let Some(turso) = state.server.turso_store_for_tenant(tenant).await { - temper_platform::persist_agent_verification(&turso, tenant, &hashes, &cache).await; + if let Some(store) = state.server.platform_store_for_tenant(tenant).await { + temper_platform::persist_agent_verification(store.as_ref(), tenant, &hashes, &cache) + .await; } } // In TenantRouted mode, bootstrap agent specs for all registered tenants. @@ -474,8 +487,14 @@ pub(super) async fn bootstrap_tenants(state: &PlatformState, apps: &[(String, St for tenant in provider.connected_tenants().await { let cache = load_verified_cache(state, &tenant).await; let hashes = temper_platform::bootstrap_agent_specs(state, &tenant, true, &cache); - if let Some(turso) = state.server.turso_store_for_tenant(&tenant).await { - temper_platform::persist_agent_verification(&turso, &tenant, &hashes, &cache).await; + if let Some(store) = state.server.platform_store_for_tenant(&tenant).await { + temper_platform::persist_agent_verification( + store.as_ref(), + &tenant, + &hashes, + &cache, + ) + .await; } } } @@ -621,14 +640,180 @@ pub(super) async fn bootstrap_installed_apps( #[cfg(test)] mod tests { + use sqlx::PgPool; use temper_platform::os_apps::get_os_app; use temper_platform::state::PlatformState; + use temper_runtime::persistence::EventStore; use temper_runtime::tenant::TenantId; use temper_server::storage::StorageStack; use temper_spec::csdl::parse_csdl; + use temper_store_postgres::{PostgresEventStore, PostgresSpecVerificationUpdate}; use temper_store_turso::TursoEventStore; - use super::bootstrap_installed_apps; + use super::{bootstrap_installed_apps, load_verified_cache, registered_hydration_tenants}; + + #[test] + fn restored_registry_tenants_are_hydrated_without_cli_apps() { + let tenant = "restored-postgres-tenant"; + let bundle = get_os_app("temper-fs").expect("temper-fs bundle"); + let csdl_xml = bundle.csdl.clone().expect("temper-fs CSDL"); + let csdl = parse_csdl(&csdl_xml).expect("parse CSDL"); + let refs = bundle + .specs + .iter() + .map(|(entity_type, source)| (entity_type.as_str(), source.as_str())) + .collect::>(); + let state = PlatformState::new(None); + state + .registry + .write() + .expect("registry") + .register_tenant(tenant, csdl, csdl_xml, &refs); + + assert_eq!( + registered_hydration_tenants(&state), + [TenantId::from(tenant)].into_iter().collect(), + "startup must hydrate tenants restored from PostgreSQL even with no --app" + ); + } + + #[tokio::test] + async fn postgres_agent_bootstrap_republishes_replacement_tombstone() { + let Ok(database_url) = std::env::var("DATABASE_URL") else { + return; + }; + let pool = PgPool::connect(&database_url) + .await + .expect("connect Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("migrate Postgres"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("bootstrap-agent-postgres-{}", uuid::Uuid::new_v4()); + let legacy_fingerprint = temper_store_turso::spec_content_hash("legacy Agent declaration"); + let unrelated_a = "[automaton]\nname = \"Unrelated\"\n# committed-a\n"; + let unrelated_b = "[automaton]\nname = \"Unrelated\"\n# staged-b\n"; + let unrelated_a_fingerprint = temper_store_turso::spec_content_hash(unrelated_a); + let unrelated_b_fingerprint = temper_store_turso::spec_content_hash(unrelated_b); + + store + .begin_vector_index_reconciliation( + &tenant, + "Agent", + "v2|legacy", + 1, + &legacy_fingerprint, + ) + .await + .expect("bootstrap compatibility authority"); + store + .persist_spec_catalog_update(&tenant, &[], "", &[], true, None) + .await + .expect("replacement tombstones compatibility authority"); + + store + .upsert_spec( + &tenant, + "Unrelated", + unrelated_a, + "", + &unrelated_a_fingerprint, + ) + .await + .expect("stage unrelated A"); + store + .commit_verified_spec( + &tenant, + "Unrelated", + &unrelated_a_fingerprint, + "", + PostgresSpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .expect("commit unrelated A"); + store + .upsert_spec( + &tenant, + "Unrelated", + unrelated_b, + "", + &unrelated_b_fingerprint, + ) + .await + .expect("stage unrelated B during built-in bootstrap window"); + + let mut state = PlatformState::new(None); + state + .server + .set_storage_stack(StorageStack::from_postgres(store)); + assert!(state.server.turso_store_for_tenant(&tenant).await.is_none()); + let cache = load_verified_cache(&state, &tenant).await; + let hashes = temper_platform::bootstrap_agent_specs(&state, &tenant, true, &cache); + let platform_store = state + .server + .platform_store_for_tenant(&tenant) + .await + .expect("Postgres must provide tenant platform persistence"); + temper_platform::persist_agent_verification( + platform_store.as_ref(), + &tenant, + &hashes, + &cache, + ) + .await; + + let expected_agent_fingerprint = hashes + .iter() + .find(|(entity_type, _)| entity_type == "Agent") + .map(|(_, fingerprint)| fingerprint) + .expect("Agent bootstrap fingerprint"); + let authority: (String, bool) = sqlx::query_as( + "SELECT declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Agent'", + ) + .bind(&tenant) + .fetch_one(&pool) + .await + .expect("re-published Agent authority"); + assert_eq!(&authority.0, expected_agent_fingerprint); + assert!(authority.1); + + let unrelated_catalog: (String, bool) = sqlx::query_as( + "SELECT content_hash, committed FROM specs \ + WHERE tenant = $1 AND entity_type = 'Unrelated'", + ) + .bind(&tenant) + .fetch_one(&pool) + .await + .expect("read committed unrelated A after built-in bootstrap"); + assert_eq!(unrelated_catalog, (unrelated_a_fingerprint.clone(), true)); + let unrelated_staging: (String,) = sqlx::query_as( + "SELECT content_hash FROM staged_specs \ + WHERE tenant = $1 AND entity_type = 'Unrelated'", + ) + .bind(&tenant) + .fetch_one(&pool) + .await + .expect("read unrelated staging B after built-in bootstrap"); + assert_eq!(unrelated_staging.0, unrelated_b_fingerprint); + let unrelated_authority: (String, bool) = sqlx::query_as( + "SELECT declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Unrelated'", + ) + .bind(&tenant) + .fetch_one(&pool) + .await + .expect("read unrelated authority after built-in bootstrap"); + assert_eq!(unrelated_authority, (unrelated_a_fingerprint, true)); + } #[tokio::test] async fn bootstrap_installed_apps_replays_persisted_app_when_registry_specs_are_stale() { diff --git a/crates/temper-cli/src/serve/storage.rs b/crates/temper-cli/src/serve/storage.rs index 3f831faa5..2fd20734a 100644 --- a/crates/temper-cli/src/serve/storage.rs +++ b/crates/temper-cli/src/serve/storage.rs @@ -52,51 +52,32 @@ pub(super) async fn upsert_loaded_specs_to_postgres( tenant: &str, loaded: &LoadedTenantSpecs, ) -> Result<()> { - for (entity_type, ioa_source) in &loaded.ioa_sources { - sqlx::query( - "INSERT INTO specs \ - (tenant, entity_type, ioa_source, csdl_xml, version, verified, verification_status, updated_at) \ - VALUES ($1, $2, $3, $4, 1, false, 'pending', now()) \ - ON CONFLICT (tenant, entity_type) DO UPDATE SET \ - ioa_source = EXCLUDED.ioa_source, \ - csdl_xml = EXCLUDED.csdl_xml, \ - version = specs.version + 1, \ - verified = false, \ - verification_status = 'pending', \ - levels_passed = NULL, \ - levels_total = NULL, \ - verification_result = NULL, \ - updated_at = now()", + let fingerprints = loaded + .ioa_sources + .iter() + .map(|(entity_type, ioa_source)| { + ( + entity_type.as_str(), + ioa_source.as_str(), + temper_store_turso::spec_content_hash(ioa_source), + ) + }) + .collect::>(); + let specs = fingerprints + .iter() + .map(|(entity_type, source, fingerprint)| (*entity_type, *source, fingerprint.as_str())) + .collect::>(); + PostgresEventStore::new(pool.clone()) + .persist_spec_catalog_update( + tenant, + &specs, + &loaded.csdl_xml, + &[], + true, + loaded.cross_invariants_toml.as_deref(), ) - .bind(tenant) - .bind(entity_type) - .bind(ioa_source) - .bind(&loaded.csdl_xml) - .execute(pool) .await - .with_context(|| format!("Failed to persist spec {tenant}/{entity_type}"))?; - } - if let Some(source) = loaded.cross_invariants_toml.as_deref() { - sqlx::query( - "INSERT INTO tenant_constraints (tenant, cross_invariants_toml, version, updated_at) \ - VALUES ($1, $2, 1, now()) \ - ON CONFLICT (tenant) DO UPDATE SET \ - cross_invariants_toml = EXCLUDED.cross_invariants_toml, \ - version = tenant_constraints.version + 1, \ - updated_at = now()", - ) - .bind(tenant) - .bind(source) - .execute(pool) - .await - .with_context(|| format!("Failed to persist tenant constraints for {tenant}"))?; - } else { - sqlx::query("DELETE FROM tenant_constraints WHERE tenant = $1") - .bind(tenant) - .execute(pool) - .await - .with_context(|| format!("Failed to clear tenant constraints for {tenant}"))?; - } + .with_context(|| format!("Failed to persist spec catalog for {tenant} in Postgres"))?; Ok(()) } @@ -110,26 +91,32 @@ pub(super) async fn upsert_loaded_specs_to_turso( tenant: &str, loaded: &LoadedTenantSpecs, ) -> Result<()> { - for (entity_type, ioa_source) in &loaded.ioa_sources { - let hash = temper_store_turso::spec_content_hash(ioa_source); - turso - .upsert_spec(tenant, entity_type, ioa_source, &loaded.csdl_xml, &hash) - .await - .with_context(|| format!("Failed to persist spec {tenant}/{entity_type} in Turso"))?; - } - if let Some(source) = loaded.cross_invariants_toml.as_deref() { - turso - .upsert_tenant_constraints(tenant, source) - .await - .with_context(|| { - format!("Failed to persist tenant constraints for {tenant} in Turso") - })?; - } else { - turso - .delete_tenant_constraints(tenant) - .await - .with_context(|| format!("Failed to clear tenant constraints for {tenant} in Turso"))?; - } + let fingerprints = loaded + .ioa_sources + .iter() + .map(|(entity_type, ioa_source)| { + ( + entity_type.as_str(), + ioa_source.as_str(), + temper_store_turso::spec_content_hash(ioa_source), + ) + }) + .collect::>(); + let specs = fingerprints + .iter() + .map(|(entity_type, source, fingerprint)| (*entity_type, *source, fingerprint.as_str())) + .collect::>(); + turso + .persist_spec_catalog_update( + tenant, + &specs, + &loaded.csdl_xml, + &[], + true, + loaded.cross_invariants_toml.as_deref(), + ) + .await + .with_context(|| format!("Failed to persist spec catalog for {tenant} in Turso"))?; if let Some(policy_text) = loaded.cedar_policy_text.as_deref() { turso .save_policy(tenant, "primary", policy_text, "system") diff --git a/crates/temper-jit/Cargo.toml b/crates/temper-jit/Cargo.toml index 39ccbeb70..6a449262e 100644 --- a/crates/temper-jit/Cargo.toml +++ b/crates/temper-jit/Cargo.toml @@ -11,6 +11,7 @@ temper-runtime = { workspace = true } temper-spec = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/temper-jit/src/shadow.rs b/crates/temper-jit/src/shadow.rs index 8cd41174f..0d8da03ac 100644 --- a/crates/temper-jit/src/shadow.rs +++ b/crates/temper-jit/src/shadow.rs @@ -113,6 +113,7 @@ mod tests { fn base_table() -> TransitionTable { let mut table = TransitionTable { entity_name: "Order".into(), + spec_declaration_fingerprint: None, states: vec!["Draft".into(), "Submitted".into(), "Cancelled".into()], initial_state: "Draft".into(), keys: vec![], diff --git a/crates/temper-jit/src/swap.rs b/crates/temper-jit/src/swap.rs index 6b05c59f6..823422e52 100644 --- a/crates/temper-jit/src/swap.rs +++ b/crates/temper-jit/src/swap.rs @@ -84,6 +84,7 @@ mod tests { fn dummy_table(name: &str) -> TransitionTable { let mut table = TransitionTable { entity_name: name.to_string(), + spec_declaration_fingerprint: None, states: vec!["A".into(), "B".into()], initial_state: "A".into(), keys: vec![], diff --git a/crates/temper-jit/src/table/builder.rs b/crates/temper-jit/src/table/builder.rs index bf48afc67..371d47b1f 100644 --- a/crates/temper-jit/src/table/builder.rs +++ b/crates/temper-jit/src/table/builder.rs @@ -4,6 +4,7 @@ //! translation layer in `temper-spec`. The shared layer eliminates duplicated //! guard/effect translation logic between JIT and verification paths. +use sha2::{Digest, Sha256}; use temper_spec::automaton::{self, Automaton, ResolvedEffect, ResolvedGuard, translate_actions}; use super::guard::Guard; @@ -21,7 +22,11 @@ impl TransitionTable { pub fn try_from_ioa_source(ioa_toml: &str) -> Result { let automaton = automaton::parse_automaton(ioa_toml) .map_err(|e| format!("failed to parse I/O Automaton TOML: {e}"))?; - Ok(Self::from_automaton(&automaton)) + let mut table = Self::from_automaton(&automaton); + let mut hasher = Sha256::new(); + hasher.update(ioa_toml.as_bytes()); + table.spec_declaration_fingerprint = Some(format!("{:x}", hasher.finalize())); + Ok(table) } /// Build a TransitionTable from I/O Automaton TOML source. @@ -118,6 +123,7 @@ impl TransitionTable { TransitionTable { entity_name: automaton.automaton.name.clone(), + spec_declaration_fingerprint: None, states: automaton.automaton.states.clone(), initial_state: automaton.automaton.initial.clone(), rules, diff --git a/crates/temper-jit/src/table/types.rs b/crates/temper-jit/src/table/types.rs index e18fe1ffe..1edb19045 100644 --- a/crates/temper-jit/src/table/types.rs +++ b/crates/temper-jit/src/table/types.rs @@ -44,6 +44,13 @@ pub struct DeclaredVector { pub struct TransitionTable { /// The entity this table governs (e.g. "Order"). pub entity_name: String, + /// SHA-256 fingerprint of the IOA source compiled into this table. + /// + /// Tables built directly from an already-parsed automaton may not have a + /// source fingerprint. Persistent entity writes require tables built from + /// IOA source so stores can reject stale-replica index rows atomically. + #[serde(default)] + pub spec_declaration_fingerprint: Option, /// All valid state values. pub states: Vec, /// The state an entity starts in. @@ -152,6 +159,8 @@ impl<'de> Deserialize<'de> for TransitionTable { #[derive(Deserialize)] struct TransitionTableRaw { entity_name: String, + #[serde(default)] + spec_declaration_fingerprint: Option, states: Vec, initial_state: String, rules: Vec, @@ -168,6 +177,7 @@ impl<'de> Deserialize<'de> for TransitionTable { let raw = TransitionTableRaw::deserialize(deserializer)?; let mut table = TransitionTable { entity_name: raw.entity_name, + spec_declaration_fingerprint: raw.spec_declaration_fingerprint, states: raw.states, initial_state: raw.initial_state, rules: raw.rules, @@ -284,6 +294,7 @@ mod tests { fn rebuild_index_groups_by_name() { let mut table = TransitionTable { entity_name: "TestEntity".to_string(), + spec_declaration_fingerprint: None, states: vec!["Draft".to_string(), "Active".to_string()], initial_state: "Draft".to_string(), keys: vec![], diff --git a/crates/temper-platform/src/bootstrap.rs b/crates/temper-platform/src/bootstrap.rs index 8cbbdd176..75a74d31b 100644 --- a/crates/temper-platform/src/bootstrap.rs +++ b/crates/temper-platform/src/bootstrap.rs @@ -293,7 +293,6 @@ pub(crate) async fn persist_bootstrap_verification( verified_cache: &BTreeMap, ) { let hashes_to_persist = hashes_requiring_persistence(hashes, verified_cache); - let mut wrote_specs = false; for (entity_type, content_hash) in &hashes_to_persist { // Find the IOA source for this entity type. @@ -311,13 +310,16 @@ pub(crate) async fn persist_bootstrap_verification( tracing::warn!("Failed to persist bootstrap spec {tenant}/{entity_type}: {e}"); continue; } - wrote_specs = true; - // Mark as verified (bootstrap panics on failure, so all specs here passed). + // Atomically publish verification for exactly the bytes that passed. + // Another replica may stage the same tenant/type between the upsert + // above and this call; the expected fingerprint makes that fail closed. if let Err(e) = store - .persist_spec_verification( + .commit_verified_spec( tenant, entity_type, + content_hash, + csdl_source, SpecVerificationUpdate { status: "completed", verified: true, @@ -328,16 +330,9 @@ pub(crate) async fn persist_bootstrap_verification( ) .await { - tracing::warn!("Failed to persist verification status for {tenant}/{entity_type}: {e}"); + tracing::warn!("Failed to commit verified bootstrap spec {tenant}/{entity_type}: {e}"); } } - - // `upsert_spec` marks rows as uncommitted while content is rewritten. Once - // bootstrap verification succeeds, promote the tenant's spec set back to a - // durable committed state so restart recovery can actually see the rows. - if wrote_specs && let Err(e) = store.commit_specs(tenant).await { - tracing::warn!("Failed to commit bootstrap specs for tenant '{tenant}': {e}"); - } } fn hashes_requiring_persistence( diff --git a/crates/temper-platform/src/os_apps/mod.rs b/crates/temper-platform/src/os_apps/mod.rs index b1e0b78ad..292541fcc 100644 --- a/crates/temper-platform/src/os_apps/mod.rs +++ b/crates/temper-platform/src/os_apps/mod.rs @@ -13,6 +13,7 @@ use std::time::Instant; use chrono::{DateTime, NaiveDateTime, Utc}; use serde::Serialize; use temper_runtime::tenant::TenantId; +use temper_server::platform_store::SpecCommitExpectation; use temper_server::state::WasmModuleSource; use temper_spec::automaton; use temper_spec::csdl::{emit_csdl_xml, merge_csdl, parse_csdl}; @@ -1249,6 +1250,7 @@ pub(super) async fn install_os_app_with_plan( .as_ref() .and_then(|stack| stack.platform.clone()) { + let mut expected_specs = Vec::with_capacity(bundle.specs.len()); if plan.specs && let Some(ref merged) = merged_csdl { @@ -1257,6 +1259,7 @@ pub(super) async fn install_os_app_with_plan( ps.upsert_spec(tenant, entity_type, ioa_source, merged, &hash) .await .map_err(|e| format!("Failed to persist spec {entity_type}: {e}"))?; + expected_specs.push((entity_type.as_str(), hash)); } } if let Some(ref policy_text) = combined_policy { @@ -1274,11 +1277,20 @@ pub(super) async fn install_os_app_with_plan( ps.record_installed_app(tenant, app_name) .await .map_err(|e| format!("Failed to record os-app installation: {e}"))?; - if plan.specs { - // Commit only when this path used individual spec writes. - ps.commit_specs(tenant) + if let Some(ref merged) = merged_csdl + && !expected_specs.is_empty() + { + let expected = expected_specs + .iter() + .map(|(entity_type, hash)| SpecCommitExpectation { + entity_type, + content_hash: hash, + csdl_xml: merged, + }) + .collect::>(); + ps.commit_spec_batch(tenant, &expected) .await - .map_err(|e| format!("Failed to commit specs: {e}"))?; + .map_err(|e| format!("Failed to commit app spec batch: {e}"))?; } } diff --git a/crates/temper-runtime/src/actor/actor_ref.rs b/crates/temper-runtime/src/actor/actor_ref.rs index 28802a23c..2eb4d49e7 100644 --- a/crates/temper-runtime/src/actor/actor_ref.rs +++ b/crates/temper-runtime/src/actor/actor_ref.rs @@ -1,4 +1,6 @@ use std::fmt; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; use tokio::sync::oneshot; @@ -38,6 +40,9 @@ pub enum SystemSignal { pub struct ActorRef { pub(crate) sender: MailboxSender, pub(crate) id: ActorId, + /// Packed supervised-incarnation state. The low bit is readiness and the + /// remaining bits are a monotonically increasing `pre_start` epoch. + pub(crate) lifecycle: Arc, } /// Unique identifier for an actor instance. @@ -111,6 +116,21 @@ impl ActorRef { &self.id } + /// Whether this actor incarnation completed `pre_start` and is serving messages. + pub fn is_ready(&self) -> bool { + self.ready_incarnation().is_some() + } + + /// Return the ready supervised-incarnation epoch in one atomic observation. + /// + /// An [`ActorId`] identifies the mailbox/task. Supervision can run + /// `pre_start` repeatedly inside that task, so callers that must distinguish + /// initialized state across a restart also need this epoch. + pub fn ready_incarnation(&self) -> Option { + let lifecycle = self.lifecycle.load(Ordering::Acquire); + (lifecycle & 1 == 1).then_some(lifecycle >> 1) + } + /// Current in-flight mailbox depth (messages queued but not yet processed). /// Exposed for observability; see `runtime_metrics::record_actor_mailbox_depth`. pub fn mailbox_depth(&self) -> usize { @@ -133,6 +153,7 @@ impl Clone for ActorRef { Self { sender: self.sender.clone(), id: self.id.clone(), + lifecycle: self.lifecycle.clone(), } } } diff --git a/crates/temper-runtime/src/actor/cell.rs b/crates/temper-runtime/src/actor/cell.rs index 5bbca1c24..bd5466149 100644 --- a/crates/temper-runtime/src/actor/cell.rs +++ b/crates/temper-runtime/src/actor/cell.rs @@ -1,3 +1,6 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + use tracing::{error, info, warn}; use super::actor_ref::{ActorId, ActorRef, Envelope, SystemSignal}; @@ -16,6 +19,53 @@ pub struct ActorCell { mailbox_capacity: usize, } +/// Clears publication readiness whenever the actor task future is dropped. +/// +/// Normal shutdown, panic unwinding, and task cancellation all drop the run +/// future, so a dead incarnation can never remain externally marked ready. +struct ActorReadiness { + lifecycle: Arc, +} + +impl ActorReadiness { + fn new(lifecycle: Arc) -> Self { + lifecycle.store(0, Ordering::Release); + Self { lifecycle } + } + + fn begin_incarnation(&self) { + let current = self.lifecycle.load(Ordering::Acquire); + assert_eq!( + current & 1, + 0, + "actor cannot begin a supervised incarnation while marked ready" + ); + let next = current + .checked_add(2) + .expect("actor supervised-incarnation epoch exhausted"); + self.lifecycle.store(next, Ordering::Release); + } + + fn mark_ready(&self) { + let previous = self.lifecycle.fetch_or(1, Ordering::AcqRel); + assert_eq!( + previous & 1, + 0, + "actor supervised incarnation was already marked ready" + ); + } + + fn mark_unready(&self) { + self.lifecycle.fetch_and(!1, Ordering::AcqRel); + } +} + +impl Drop for ActorReadiness { + fn drop(&mut self) { + self.mark_unready(); + } +} + impl ActorCell { /// Create a new actor cell with the given actor and ID. pub fn new(actor: A, id: ActorId) -> Self { @@ -36,13 +86,15 @@ impl ActorCell { pub fn spawn(self) -> ActorRef { let (tx, rx) = mailbox::mailbox(self.mailbox_capacity); let id = self.id.clone(); + let lifecycle = Arc::new(AtomicU64::new(0)); let actor_ref = ActorRef { sender: tx, id: id.clone(), + lifecycle: lifecycle.clone(), }; - tokio::spawn(self.run(rx)); // determinism-ok: production actor cell, not on simulation path + tokio::spawn(self.run(rx, lifecycle)); // determinism-ok: production actor cell, not on simulation path actor_ref } @@ -51,7 +103,8 @@ impl ActorCell { /// 1. pre_start → initialize state /// 2. loop: receive message → handle /// 3. post_stop → cleanup - async fn run(self, mut rx: MailboxReceiver) { + async fn run(self, mut rx: MailboxReceiver, lifecycle: Arc) { + let readiness = ActorReadiness::new(lifecycle); let actor = self.actor; let id = self.id; let strategy = actor.supervision_strategy(); @@ -59,6 +112,8 @@ impl ActorCell { let mut restart_count: u32 = 0; loop { + readiness.mark_unready(); + readiness.begin_incarnation(); // Phase 1: Initialize let mut ctx = ActorContext::new(id.clone()); info!(actor = %id, "actor starting"); @@ -67,6 +122,7 @@ impl ActorCell { Ok(s) => { info!(actor = %id, "actor started"); restart_count = 0; + readiness.mark_ready(); s } Err(e) => { @@ -133,6 +189,7 @@ impl ActorCell { }; // Phase 3: Cleanup + readiness.mark_unready(); info!(actor = %id, "actor stopping"); actor.post_stop(state, &mut ctx).await; @@ -159,7 +216,122 @@ fn should_restart(strategy: &SupervisionStrategy, current_restarts: u32) -> bool #[cfg(test)] mod tests { use super::*; + use crate::actor::{ActorContext, Message}; use std::time::Duration; + use tokio::sync::Notify; + + #[derive(Debug)] + enum PanickingMsg { + Crash, + } + + impl Message for PanickingMsg {} + + struct PanickingActor { + started: Arc, + } + + impl Actor for PanickingActor { + type Msg = PanickingMsg; + type State = (); + + async fn pre_start( + &self, + _ctx: &mut ActorContext, + ) -> Result { + self.started.notify_one(); + Ok(()) + } + + async fn handle( + &self, + msg: Self::Msg, + _state: &mut Self::State, + _ctx: &mut ActorContext, + ) -> Result<(), ActorError> { + match msg { + PanickingMsg::Crash => panic!("intentional handler panic"), + } + } + + async fn post_stop(&self, _state: Self::State, _ctx: &mut ActorContext) {} + } + + #[tokio::test] + async fn handler_panic_clears_actor_readiness() { + let started = Arc::new(Notify::new()); + let actor = ActorCell::new( + PanickingActor { + started: started.clone(), + }, + ActorId::new("panicking", "system/panicking"), + ) + .spawn(); + started.notified().await; + tokio::time::timeout(Duration::from_secs(1), async { + while !actor.is_ready() { + tokio::task::yield_now().await; + } + }) + .await + .expect("actor must publish readiness after pre_start"); + + actor + .tell(PanickingMsg::Crash) + .expect("enqueue crashing message"); + tokio::time::timeout(Duration::from_secs(1), async { + while actor.is_ready() { + tokio::task::yield_now().await; + } + }) + .await + .expect("actor panic must clear readiness through the run-future drop guard"); + assert!(!actor.is_ready()); + } + + #[tokio::test] + async fn supervised_restart_advances_ready_incarnation_without_changing_actor_id() { + let started = Arc::new(Notify::new()); + let actor = ActorCell::new( + PanickingActor { + started: started.clone(), + }, + ActorId::new("restarting", "system/restarting"), + ) + .spawn(); + started.notified().await; + let first_incarnation = tokio::time::timeout(Duration::from_secs(1), async { + loop { + if let Some(incarnation) = actor.ready_incarnation() { + break incarnation; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("actor must publish its first ready incarnation"); + let actor_uid = actor.id().uid; + + actor + .signal(SystemSignal::Restart) + .expect("enqueue supervised restart"); + let second_incarnation = tokio::time::timeout(Duration::from_secs(1), async { + loop { + if let Some(incarnation) = actor.ready_incarnation() + && incarnation != first_incarnation + { + break incarnation; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("actor must publish its restarted incarnation"); + + assert_eq!(actor.id().uid, actor_uid); + assert!(second_incarnation > first_incarnation); + actor.stop().expect("stop restarted actor"); + } #[test] fn stop_strategy_never_restarts() { diff --git a/crates/temper-runtime/src/persistence/indexing.rs b/crates/temper-runtime/src/persistence/indexing.rs new file mode 100644 index 000000000..f95fb4e9f --- /dev/null +++ b/crates/temper-runtime/src/persistence/indexing.rs @@ -0,0 +1,106 @@ +use serde::{Deserialize, Serialize}; + +use super::PersistenceEnvelope; + +/// A declared-key row to co-commit with an append (ADR-0153). The entity claims +/// `key_hash` for `key_name`; the store writes it into `entity_key_index` in the +/// same transaction as the journal append, giving the read plane an `O(log n)` +/// present/absent probe (the negative-existence access path, ARN-68). +#[derive(Debug, Clone)] +pub struct EntityKeyRow { + /// The declared key's identifier (the `[[key]]` block's `name`). + pub key_name: String, + /// The canonical, type-tagged hash of the key's values. + pub key_hash: String, +} + +/// A derived vector-index row to co-commit with an append (ADR-0155). Parsed from +/// the entity's post-transition state for one declared `[[vector]]` path: the +/// float vector and the model tag that partitions its space. Stores that maintain +/// `entity_vector_index` write one row per `(decl_name, model_tag, entity_id)`; the +/// blob is packed little-endian f32. Unlike a key row this has no uniqueness +/// constraint — it is derived, rebuildable ranking state. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EntityVectorRow { + /// The declared vector path's identifier (the `[[vector]]` block's `name`). + pub decl_name: String, + /// The model tag that partitions this vector's space (only same-tag vectors + /// are ever compared). + pub model_tag: String, + /// The float vector, exactly `dims` long. + pub vector: Vec, +} + +/// Pack an `f32` slice to little-endian bytes — the `entity_vector_index` blob +/// encoding shared by every backend (ADR-0155). Kept here beside [`EntityVectorRow`] +/// so the stores and the kernel ranking agree on the byte layout. +pub fn pack_f32_le(vector: &[f32]) -> Vec { + let mut bytes = Vec::with_capacity(vector.len() * 4); + for value in vector { + bytes.extend_from_slice(&value.to_le_bytes()); + } + bytes +} + +/// Unpack little-endian bytes back to `f32`. `None` if the byte length is not a +/// multiple of 4, or if any component is not finite (both signal a corrupt blob), +/// so a bad row is skipped rather than panicking or feeding a `NaN`/`inf` into the +/// kNN ranking — where a `NaN` would sort ahead of every real score. +pub fn unpack_f32_le(bytes: &[u8]) -> Option> { + if !bytes.len().is_multiple_of(4) { + return None; + } + let mut out = Vec::with_capacity(bytes.len() / 4); + for chunk in bytes.chunks_exact(4) { + let value = f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); + if !value.is_finite() { + return None; + } + out.push(value); + } + Some(out) +} + +/// One candidate row returned from the vector index for a kNN read (ADR-0155): +/// an entity and its packed vector for one `(tenant, type, decl, model_tag)` +/// partition. The kernel — not the store — computes the metric over these in the +/// store-supplied (entity-id) order, so ranking is identical across backends. +#[derive(Debug, Clone, PartialEq)] +pub struct EntityVectorCandidate { + /// The entity holding this vector. + pub entity_id: String, + /// The float vector, exactly `dims` long. + pub vector: Vec, +} + +/// One stream append inside an atomic multi-journal append. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PersistenceAppend { + /// Persistence ID in the form `{tenant}:{entity_type}:{entity_id}`. + pub persistence_id: String, + /// Optimistic-concurrency sequence expected before this append. + pub expected_sequence: u64, + /// Events to append to this journal. + pub events: Vec, + /// Complete post-transition vector rows to co-commit for this stream. + #[serde(default)] + pub vector_rows: Vec, + /// Whether this stream's type declares vectors. When true, an empty + /// `vector_rows` purges candidates while retaining the live-write fence. + #[serde(default)] + pub reconcile_vectors: bool, + /// SHA-256 fingerprint of the IOA source that produced this stream's + /// post-transition index rows. Durable stores validate it against the spec + /// catalog in the same transaction as the journal append. + #[serde(default)] + pub spec_declaration_fingerprint: Option, +} + +/// New sequence number for one stream after an atomic multi-journal append. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PersistenceAppendResult { + /// Persistence ID that was appended. + pub persistence_id: String, + /// New highest sequence number for this journal. + pub sequence_nr: u64, +} diff --git a/crates/temper-runtime/src/persistence/mod.rs b/crates/temper-runtime/src/persistence/mod.rs index 80c236adc..e7b67a222 100644 --- a/crates/temper-runtime/src/persistence/mod.rs +++ b/crates/temper-runtime/src/persistence/mod.rs @@ -1,5 +1,16 @@ use serde::{Deserialize, Serialize}; +mod indexing; +pub use indexing::{ + EntityKeyRow, EntityVectorCandidate, EntityVectorRow, PersistenceAppend, + PersistenceAppendResult, pack_f32_le, unpack_f32_le, +}; +mod types; +pub use types::{ + CompositeEvent, CompositeEventSubWrite, EventMetadata, PersistenceEnvelope, PersistenceError, + storage_error, +}; + /// Event type used for the parent-journal record of a Composite action. /// /// Concrete sub-write events remain the state-changing events on their target @@ -7,27 +18,6 @@ use serde::{Deserialize, Serialize}; /// journals/idempotency keys that were committed atomically with it. pub const COMPOSITE_EVENT_TYPE: &str = "CompositeEvent"; -/// Replay/audit record for one Composite action application. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct CompositeEvent { - pub tenant: String, - pub parent_entity_type: String, - pub parent_entity_id: String, - pub parent_action: String, - pub composite_idempotency_key: String, - pub sub_writes: Vec, -} - -/// One concrete sub-write recorded in a [`CompositeEvent`]. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct CompositeEventSubWrite { - pub index: usize, - pub entity_type: String, - pub entity_id: String, - pub action: String, - pub idempotency_key: String, -} - /// Marker trait for domain events. /// Events must be serializable (for persistence) and Send + 'static (for async). pub trait DomainEvent: @@ -35,21 +25,6 @@ pub trait DomainEvent: { } -/// Metadata attached to every persisted event. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct EventMetadata { - /// Unique ID of this event. - pub event_id: uuid::Uuid, - /// ID of the command/message that caused this event. - pub causation_id: uuid::Uuid, - /// Correlation ID for tracing across actor boundaries. - pub correlation_id: uuid::Uuid, - /// Timestamp of persistence. - pub timestamp: chrono::DateTime, - /// Actor that produced this event. - pub actor_id: String, -} - /// Trait for event-sourced persistent actors. /// Extends the base Actor trait with event journal and snapshot capabilities. /// @@ -83,77 +58,6 @@ pub trait PersistentActor: Send + 'static { } } -/// A declared-key row to co-commit with an append (ADR-0153). The entity claims -/// `key_hash` for `key_name`; the store writes it into `entity_key_index` in the -/// same transaction as the journal append, giving the read plane an `O(log n)` -/// present/absent probe (the negative-existence access path, ARN-68). -#[derive(Debug, Clone)] -pub struct EntityKeyRow { - /// The declared key's identifier (the `[[key]]` block's `name`). - pub key_name: String, - /// The canonical, type-tagged hash of the key's values. - pub key_hash: String, -} - -/// A derived vector-index row to co-commit with an append (ADR-0155). Parsed from -/// the entity's post-transition state for one declared `[[vector]]` path: the -/// float vector and the model tag that partitions its space. Stores that maintain -/// `entity_vector_index` write one row per `(decl_name, model_tag, entity_id)`; the -/// blob is packed little-endian f32. Unlike a key row this has no uniqueness -/// constraint — it is derived, rebuildable ranking state. -#[derive(Debug, Clone, PartialEq)] -pub struct EntityVectorRow { - /// The declared vector path's identifier (the `[[vector]]` block's `name`). - pub decl_name: String, - /// The model tag that partitions this vector's space (only same-tag vectors - /// are ever compared). - pub model_tag: String, - /// The float vector, exactly `dims` long. - pub vector: Vec, -} - -/// Pack an `f32` slice to little-endian bytes — the `entity_vector_index` blob -/// encoding shared by every backend (ADR-0155). Kept here beside [`EntityVectorRow`] -/// so the stores and the kernel ranking agree on the byte layout. -pub fn pack_f32_le(vector: &[f32]) -> Vec { - let mut bytes = Vec::with_capacity(vector.len() * 4); - for value in vector { - bytes.extend_from_slice(&value.to_le_bytes()); - } - bytes -} - -/// Unpack little-endian bytes back to `f32`. `None` if the byte length is not a -/// multiple of 4, or if any component is not finite (both signal a corrupt blob), -/// so a bad row is skipped rather than panicking or feeding a `NaN`/`inf` into the -/// kNN ranking — where a `NaN` would sort ahead of every real score. -pub fn unpack_f32_le(bytes: &[u8]) -> Option> { - if !bytes.len().is_multiple_of(4) { - return None; - } - let mut out = Vec::with_capacity(bytes.len() / 4); - for chunk in bytes.chunks_exact(4) { - let value = f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); - if !value.is_finite() { - return None; - } - out.push(value); - } - Some(out) -} - -/// One candidate row returned from the vector index for a kNN read (ADR-0155): -/// an entity and its packed vector for one `(tenant, type, decl, model_tag)` -/// partition. The kernel — not the store — computes the metric over these in the -/// store-supplied (entity-id) order, so ranking is identical across backends. -#[derive(Debug, Clone, PartialEq)] -pub struct EntityVectorCandidate { - /// The entity holding this vector. - pub entity_id: String, - /// The float vector, exactly `dims` long. - pub vector: Vec, -} - /// Trait for the event store backend (implemented by temper-store-postgres). /// Uses desugared async-in-trait to enforce Send bounds on futures. pub trait EventStore: Send + Sync + 'static { @@ -184,6 +88,7 @@ pub trait EventStore: Send + Sync + 'static { key_rows, &[], false, + None, ) } @@ -192,13 +97,19 @@ pub trait EventStore: Send + Sync + 'static { /// journal append. This is the single co-commit entry point the entity actor /// calls. The default ignores the index kinds and delegates to /// [`EventStore::append`] — stores with a query plane that co-commit (postgres, - /// sim) override it; Turso also overrides it to maintain the vector index - /// write-behind (event first, index follows). When `reconcile_vectors` is true + /// sim, Turso) override it. When `reconcile_vectors` is true /// (the entity's type declares ≥1 `[[vector]]` path) the store first DELETES all /// of the entity's vector rows, then inserts `vector_rows` — so a delete /// transition or a cleared vector/model property purges the stale rows instead of /// leaving them to be ranked forever. The sequence and atomicity contract is - /// identical to `append`. + /// identical to `append`. `spec_declaration_fingerprint` binds the writer's + /// compiled table to durable spec authority; indexing stores reject a stale + /// fingerprint before advancing the journal. Callers that reconcile vectors + /// must always provide it. + #[expect( + clippy::too_many_arguments, + reason = "journal, key, vector, and declaration data form one atomic storage boundary" + )] fn append_with_index_rows( &self, persistence_id: &str, @@ -207,27 +118,97 @@ pub trait EventStore: Send + Sync + 'static { key_rows: &[EntityKeyRow], vector_rows: &[EntityVectorRow], reconcile_vectors: bool, + spec_declaration_fingerprint: Option<&str>, ) -> impl std::future::Future> + Send { - let _ = (key_rows, vector_rows, reconcile_vectors); + let _ = ( + key_rows, + vector_rows, + reconcile_vectors, + spec_declaration_fingerprint, + ); self.append(persistence_id, expected_sequence, events) } + /// Persist one spec declaration fingerprint or absence tombstone. + /// + /// SQL stores derive this authority from their transactional spec catalog. + /// Deterministic stores override this hook so the production hot-load path + /// drives the same authority before publishing a rebuilt registry. + fn persist_spec_declaration( + &self, + tenant: &str, + entity_type: &str, + declaration_fingerprint: &str, + ) -> impl std::future::Future> + Send { + let _ = (tenant, entity_type, declaration_fingerprint); + async { Ok(0) } + } + + /// Return entity types whose durable declarations are currently present. + /// + /// The default is empty because SQL-backed servers enumerate their catalog + /// through the metadata store. Deterministic stores override this for + /// replacement retry/restart parity. + fn spec_declaration_entity_types( + &self, + tenant: &str, + ) -> impl std::future::Future, PersistenceError>> + Send { + let _ = tenant; + async { Ok(Vec::new()) } + } + + /// Begin reconciliation and return its durable generation (ADR-0181). + /// `declaration_revision` is monotonic; `declaration_fingerprint` identifies the + /// IOA source. Durable backends resolve both against a tombstone-preserving + /// declaration authority, so a stale caller cannot win by arriving last or after + /// delete/re-add. The returned token fences every replacement and watermark. + /// Non-indexing backends reject the operation. + fn begin_vector_index_reconciliation( + &self, + _tenant: &str, + _entity_type: &str, + _vector_set: &str, + _declaration_revision: u64, + _declaration_fingerprint: &str, + ) -> impl std::future::Future> + Send { + async { + Err(PersistenceError::Storage( + "vector-index reconciliation is unsupported by this event store".to_string(), + )) + } + } + /// Reconcile the derived vector-index rows for an **existing** entity to exactly - /// `vector_rows` (ADR-0155), without appending a journal event: DELETE every - /// existing row for `(tenant, entity_type, entity_id)`, then INSERT `vector_rows`. - /// Idempotent, and an empty `vector_rows` PURGES the entity (used to clean up a - /// deleted or un-embedded entity). Used by the backfill and by the Turso - /// write-behind path. The default is a no-op (non-indexing backends); query-plane - /// stores implement it. + /// `vector_rows` (ADR-0181), without appending a journal event. + /// `reconciliation_generation` identifies the declaration set and + /// `observed_sequence` is the journal position from which the rows were rebuilt. + /// Stores reject a generation that is no longer current, and within the current + /// generation atomically replace rows only when the sequence is at least the + /// entity's retained vector-index version. A lower sequence is a successful + /// no-op. The version survives an empty row set, so stale work cannot resurrect a + /// deleted/unembedded entity. Equal-sequence replay is idempotent. fn backfill_entity_vectors( &self, tenant: &str, entity_type: &str, entity_id: &str, + reconciliation_generation: u64, + observed_sequence: u64, vector_rows: &[EntityVectorRow], ) -> impl std::future::Future> + Send { - let _ = (tenant, entity_type, entity_id, vector_rows); - async { Ok(()) } + let _ = ( + tenant, + entity_type, + entity_id, + reconciliation_generation, + observed_sequence, + vector_rows, + ); + async { + Err(PersistenceError::Storage( + "vector-index reconciliation is unsupported by this event store".to_string(), + )) + } } /// The candidate `(entity_id, vector)` rows for one vector-index partition @@ -252,17 +233,24 @@ pub trait EventStore: Send + Sync + 'static { /// Record that `entity_vector_index` is **complete** for `(tenant, entity_type)` /// — every existing entity has had its declared vectors indexed by the backfill /// (ADR-0155 watermark, mirroring `mark_key_index_backfilled`). `vector_set` is - /// the sorted, comma-joined declared vector-path NAMES the backfill covered, so a - /// later declaration of an ADDITIONAL path is detected as a set change and the - /// type is re-indexed. Idempotent. Default no-op. + /// the revisioned signature of every covered vector declaration (name, property, + /// model property, dimensions, and metric), so any declaration change re-indexes + /// the type. The durable `reconciliation_generation` must still be current; + /// otherwise the stale completion claim is rejected. Idempotent within one + /// generation. Non-indexing backends reject the operation explicitly. fn mark_vector_index_backfilled( &self, tenant: &str, entity_type: &str, + reconciliation_generation: u64, vector_set: &str, ) -> impl std::future::Future> + Send { - let _ = (tenant, entity_type, vector_set); - async { Ok(()) } + let _ = (tenant, entity_type, reconciliation_generation, vector_set); + async { + Err(PersistenceError::Storage( + "vector-index reconciliation is unsupported by this event store".to_string(), + )) + } } /// The `(entity_type, vector_set)` watermarks for `tenant` — each type whose @@ -277,6 +265,20 @@ pub trait EventStore: Send + Sync + 'static { async { Ok(Vec::new()) } } + /// Entity types with durable vector-reconciliation state for `tenant` + /// (ADR-0181): a generation row, retained per-entity fence, or candidate row. + /// Unlike completion watermarks, this state survives an interrupted + /// reconciliation and includes generation-zero live/legacy rows. The coordinator + /// uses it as a work source so remove-all declarations cannot strand candidates. + /// Default empty for non-indexing backends. + fn vector_reconciliation_entity_types( + &self, + tenant: &str, + ) -> impl std::future::Future, PersistenceError>> + Send { + let _ = tenant; + async { Ok(Vec::new()) } + } + /// The `entity_id`s that already have at least one `entity_vector_index` row for /// `(tenant, entity_type)`. Lets the vector backfill **resume** cheaply, skipping /// already-indexed entities. Default empty (no resumption). Mirrors @@ -290,6 +292,20 @@ pub trait EventStore: Send + Sync + 'static { async { Ok(Vec::new()) } } + /// List every durable journal stream that a vector-index repair must reconcile + /// for `(tenant, entity_type)`, including deleted streams (ADR-0181). Active + /// entity listing deliberately excludes deletions on some backends, but repair + /// must retain a sequence tombstone for them so stale rows cannot survive or be + /// resurrected. Backends whose normal listing already includes the complete + /// journal set may use this default. + fn list_vector_repair_entity_ids( + &self, + tenant: &str, + entity_type: &str, + ) -> impl std::future::Future, PersistenceError>> + Send { + self.list_entity_ids_by_type(tenant, entity_type) + } + /// Backfill declared key-index rows for an **existing** entity (ADR-0153), /// without appending a journal event. Idempotent: re-running yields the same /// rows. Used to populate `entity_key_index` for entities written before the @@ -458,57 +474,3 @@ pub trait EventStore: Send + Sync + 'static { } } } - -/// A persisted event with metadata. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PersistenceEnvelope { - /// Monotonic sequence number within the entity's journal. - pub sequence_nr: u64, - /// Fully qualified event type name. - pub event_type: String, - /// Serialized event payload. - pub payload: serde_json::Value, - /// Event metadata (causation, correlation, timestamp). - pub metadata: EventMetadata, -} - -/// One stream append inside an atomic multi-journal append. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PersistenceAppend { - /// Persistence ID in the form `{tenant}:{entity_type}:{entity_id}`. - pub persistence_id: String, - /// Optimistic-concurrency sequence expected before this append. - pub expected_sequence: u64, - /// Events to append to this journal. - pub events: Vec, -} - -/// New sequence number for one stream after an atomic batch append. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct PersistenceAppendResult { - /// Persistence ID that was appended. - pub persistence_id: String, - /// New highest sequence number for this journal. - pub sequence_nr: u64, -} - -/// Errors that can occur during event persistence operations. -#[derive(Debug, thiserror::Error)] -pub enum PersistenceError { - /// Optimistic concurrency check failed (another writer appended first). - #[error("optimistic concurrency violation: expected sequence {expected}, got {actual}")] - ConcurrencyViolation { expected: u64, actual: u64 }, - - /// Event serialization or deserialization failed. - #[error("serialization error: {0}")] - Serialization(String), - - /// Underlying storage backend returned an error. - #[error("storage error: {0}")] - Storage(String), -} - -/// Convert backend-specific errors into [`PersistenceError::Storage`]. -pub fn storage_error(err: impl std::fmt::Display) -> PersistenceError { - PersistenceError::Storage(err.to_string()) -} diff --git a/crates/temper-runtime/src/persistence/types.rs b/crates/temper-runtime/src/persistence/types.rs new file mode 100644 index 000000000..1a8380610 --- /dev/null +++ b/crates/temper-runtime/src/persistence/types.rs @@ -0,0 +1,71 @@ +use serde::{Deserialize, Serialize}; + +/// Replay/audit record for one Composite action application. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CompositeEvent { + pub tenant: String, + pub parent_entity_type: String, + pub parent_entity_id: String, + pub parent_action: String, + pub composite_idempotency_key: String, + pub sub_writes: Vec, +} + +/// One concrete sub-write recorded in a [`CompositeEvent`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CompositeEventSubWrite { + pub index: usize, + pub entity_type: String, + pub entity_id: String, + pub action: String, + pub idempotency_key: String, +} + +/// Metadata attached to every persisted event. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EventMetadata { + /// Unique ID of this event. + pub event_id: uuid::Uuid, + /// ID of the command/message that caused this event. + pub causation_id: uuid::Uuid, + /// Correlation ID for tracing across actor boundaries. + pub correlation_id: uuid::Uuid, + /// Timestamp of persistence. + pub timestamp: chrono::DateTime, + /// Actor that produced this event. + pub actor_id: String, +} + +/// A persisted event with metadata. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PersistenceEnvelope { + /// Monotonic sequence number within the entity's journal. + pub sequence_nr: u64, + /// Fully qualified event type name. + pub event_type: String, + /// Serialized event payload. + pub payload: serde_json::Value, + /// Event metadata (causation, correlation, timestamp). + pub metadata: EventMetadata, +} + +/// Errors that can occur during event persistence operations. +#[derive(Debug, thiserror::Error)] +pub enum PersistenceError { + /// Optimistic concurrency check failed (another writer appended first). + #[error("optimistic concurrency violation: expected sequence {expected}, got {actual}")] + ConcurrencyViolation { expected: u64, actual: u64 }, + + /// Event serialization or deserialization failed. + #[error("serialization error: {0}")] + Serialization(String), + + /// Underlying storage backend returned an error. + #[error("storage error: {0}")] + Storage(String), +} + +/// Convert backend-specific errors into [`PersistenceError::Storage`]. +pub fn storage_error(err: impl std::fmt::Display) -> PersistenceError { + PersistenceError::Storage(err.to_string()) +} diff --git a/crates/temper-server/src/entity_actor/actor.rs b/crates/temper-server/src/entity_actor/actor.rs index cbf46c9a7..d7a4a9967 100644 --- a/crates/temper-server/src/entity_actor/actor.rs +++ b/crates/temper-server/src/entity_actor/actor.rs @@ -331,6 +331,7 @@ impl EntityActor { store: &BoxedEventStore, backend: BackendLabel, persistence_id: &str, + table: &TransitionTable, state: &mut EntityState, event: &EntityEvent, ) -> Result { @@ -354,15 +355,13 @@ impl EntityActor { // ADR-0153/0155: derive the declared key rows AND the vector-index rows from // the new state and co-commit them with the journal append, so a keyed read // is correct without a scan and a kNN read reflects the write deterministically. - let (key_rows, vector_rows, reconcile_vectors) = { - let table = self.table.read().expect("table lock poisoned"); + let (key_rows, vector_rows, reconcile_vectors, spec_declaration_fingerprint) = { // The type declares vector paths → the store reconciles this entity's // vector rows (delete stale + insert current) even when no row is emitted // this write (a delete transition or a cleared property), so stale rows are // purged instead of being ranked forever (ADR-0155). let reconcile_vectors = !table.vectors.is_empty(); let mut key_rows = Vec::new(); - let mut vector_rows = Vec::new(); if let Some(field_map) = state.fields.as_object() { for key in &table.keys { if let Some(hash) = @@ -374,36 +373,18 @@ impl EntityActor { }); } } - // A soft-deleted (tombstone) entity is never indexed — it emits no - // vector rows, so the reconcile below PURGES any it had, even though - // its embedding field may still be present. Mirrors how the field-index - // projection removes a deleted entity. - let index_vectors = state.status != "Deleted"; - for decl in table.vectors.iter().filter(|_| index_vectors) { - // A vector is indexed only when its property parses to `dims` - // floats AND its model tag is a non-empty string — otherwise the - // path indexes nothing for this entity (like an incomplete key). - let Some(vector) = field_map - .get(&decl.property) - .and_then(|v| crate::vector_index::parse_vector_property(v, decl.dims)) - else { - continue; - }; - let Some(model_tag) = field_map - .get(&decl.model_property) - .and_then(|v| v.as_str()) - .filter(|tag| !tag.is_empty()) - else { - continue; - }; - vector_rows.push(temper_runtime::persistence::EntityVectorRow { - decl_name: decl.name.clone(), - model_tag: model_tag.to_string(), - vector, - }); - } } - (key_rows, vector_rows, reconcile_vectors) + let vector_rows = crate::vector_index::rows_for_entity_state( + &table.vectors, + &event.to_status, + &state.fields, + ); + ( + key_rows, + vector_rows, + reconcile_vectors, + table.spec_declaration_fingerprint.clone(), + ) }; let append_start = Instant::now(); let result = store @@ -411,9 +392,12 @@ impl EntityActor { persistence_id, state.sequence_nr, &[envelope], - &key_rows, - &vector_rows, - reconcile_vectors, + crate::storage::AppendIndexRows { + key_rows: &key_rows, + vector_rows: &vector_rows, + reconcile_vectors, + spec_declaration_fingerprint: spec_declaration_fingerprint.as_deref(), + }, ) .await; crate::runtime_metrics::record_event_store_append_wait( @@ -504,7 +488,7 @@ impl EntityActor { state: &mut EntityState, tenant: &str, blob_store: Option<&crate::blob_store::BlobStore>, - // When true, a journal read failure PROPAGATES as an error instead of being + // When true, a journal read or envelope parse failure PROPAGATES instead of being // swallowed ("start fresh"). The key-index backfill needs this: it must // distinguish "entity genuinely has no events" from "could not read the // journal", or it would watermark a type while a present entity is unkeyed @@ -567,14 +551,23 @@ impl EntityActor { // Tombstone is terminal: once deleted, entity must not replay // into a live state. Stop at the first Deleted event. if env.event_type == "Deleted" { - let tombstone = parsed_event.unwrap_or_else(|_| EntityEvent { - action: "Deleted".to_string(), - from_status: state.status.clone(), - to_status: "Deleted".to_string(), - timestamp: env.metadata.timestamp, - params: serde_json::json!({}), - idempotency_key: None, - }); + let tombstone = match parsed_event { + Ok(event) => event, + Err(error) if strict_journal_read => { + return Err(ActorError::custom(format!( + "incompatible persisted event at sequence {} for {}:{}: {error}", + env.sequence_nr, state.entity_type, state.entity_id + ))); + } + Err(_) => EntityEvent { + action: "Deleted".to_string(), + from_status: state.status.clone(), + to_status: "Deleted".to_string(), + timestamp: env.metadata.timestamp, + params: serde_json::json!({}), + idempotency_key: None, + }, + }; state.status = tombstone.to_status.clone(); if let Some(obj) = state.fields.as_object_mut() { obj.insert( @@ -657,6 +650,12 @@ impl EntityActor { state.push_event_bounded(event); } Err(e) => { + if strict_journal_read { + return Err(ActorError::custom(format!( + "incompatible persisted event at sequence {} for {}:{}: {e}", + env.sequence_nr, state.entity_type, state.entity_id + ))); + } // Schema-mismatched event: log and skip rather than panic. // This preserves entity hydration across spec evolution — // the last valid state is used and replay continues. @@ -733,11 +732,11 @@ impl EntityActor { /// Rebuild an entity's current state from its snapshot + event tail. /// -/// `strict_journal_read`: when true, a journal read failure PROPAGATES as an error -/// instead of being swallowed into a "start fresh"/stale state. The key-index backfill -/// passes `true` so it can tell "no events" apart from "could not read the journal" — -/// keying decisions and the per-type watermark depend on that distinction (ADR-0153 -/// soundness gate). Actor hydration passes `false` (keep serving on a transient read). +/// `strict_journal_read`: when true, journal read and envelope parse failures propagate +/// instead of being swallowed into a "start fresh"/partial state. Index backfills pass +/// `true` so they can distinguish a complete replay from unreadable or incompatible +/// history before publishing a type watermark (ADR-0153/ADR-0181 soundness gate). +/// Actor hydration passes `false` to preserve compatibility during normal serving. #[allow(clippy::too_many_arguments)] pub(crate) async fn recover_entity_state_from_store( tenant: &str, @@ -812,14 +811,21 @@ impl Actor for EntityActor { if let (Some(store), Some(backend)) = (self.event_journal.as_ref(), self.event_backend) { - self.persist_event(store, backend, &self.persistence_id(), &mut state, &created) - .await - .map_err(|e| { - ActorError::custom(format!( - "failed to persist bootstrap Created event for {}:{}: {}", - self.entity_type, self.entity_id, e - )) - })?; + self.persist_event( + store, + backend, + &self.persistence_id(), + &table, + &mut state, + &created, + ) + .await + .map_err(|e| { + ActorError::custom(format!( + "failed to persist bootstrap Created event for {}:{}: {}", + self.entity_type, self.entity_id, e + )) + })?; } state.push_event_bounded(created); } @@ -1008,7 +1014,14 @@ impl Actor for EntityActor { (self.event_journal.as_ref(), self.event_backend) { let first_persist = self - .persist_event(store, backend, &self.persistence_id(), state, &event) + .persist_event( + store, + backend, + &self.persistence_id(), + &table, + state, + &event, + ) .await; match first_persist { @@ -1071,9 +1084,12 @@ impl Actor for EntityActor { state, &self.tenant, self.blob_store.as_ref(), - // Actor hydration keeps the lenient "start - // fresh on read error" behavior (unchanged). - false, + // A concurrency retry must reach the + // authoritative sequence reported by the + // rejected append. Treat an unreadable + // journal as a retry failure instead of + // continuing from an under-replayed state. + true, ) .await?; @@ -1157,6 +1173,7 @@ impl Actor for EntityActor { store, backend, &self.persistence_id(), + &table, state, &retry_event, ) @@ -1450,6 +1467,7 @@ impl Actor for EntityActor { }); } EntityMsg::Delete => { + let table = self.table.read().expect("table lock poisoned").clone(); let deleted = EntityEvent { action: "Deleted".to_string(), from_status: state.status.clone(), @@ -1462,7 +1480,14 @@ impl Actor for EntityActor { if let (Some(store), Some(backend)) = (self.event_journal.as_ref(), self.event_backend) && let Err(e) = self - .persist_event(store, backend, &self.persistence_id(), state, &deleted) + .persist_event( + store, + backend, + &self.persistence_id(), + &table, + state, + &deleted, + ) .await { ctx.reply(EntityResponse { diff --git a/crates/temper-server/src/observe/load_dir_reconciliation_test.rs b/crates/temper-server/src/observe/load_dir_reconciliation_test.rs new file mode 100644 index 000000000..26b09187d --- /dev/null +++ b/crates/temper-server/src/observe/load_dir_reconciliation_test.rs @@ -0,0 +1,523 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use temper_runtime::ActorSystem; +use temper_runtime::persistence::{EventMetadata, EventStore, PersistenceEnvelope}; +use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; +use temper_runtime::tenant::TenantId; +use temper_store_sim::SimEventStore; +use temper_store_turso::TursoEventStore; +use tower::ServiceExt; + +use crate::{EntityMsg, ServerState, SpecRegistry, StorageStack, build_router}; + +const TENANT: &str = "arn216"; +const NOTE_V1: &str = include_str!("../../tests/fixtures/arn216/full_v1/note.ioa.toml"); +const NOTE_V2: &str = include_str!("../../tests/fixtures/arn216/full_v2/note.ioa.toml"); +const CSDL_V2: &str = include_str!("../../tests/fixtures/arn216/full_v2/model.csdl.xml"); + +fn fixture(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/arn216") + .join(name) +} + +fn sim_state(store: &SimEventStore, name: &str) -> ServerState { + let mut state = ServerState::from_registry(ActorSystem::new(name), SpecRegistry::new()); + state.set_storage_stack(StorageStack::from_sim(store.clone(), None)); + state +} + +fn turso_state(store: &TursoEventStore, name: &str) -> ServerState { + let mut state = ServerState::from_registry(ActorSystem::new(name), SpecRegistry::new()); + state.set_storage_stack(StorageStack::from_turso(store.clone())); + state +} + +async fn load_dir_lines(state: &ServerState, fixture_name: &str) -> Vec { + let body = serde_json::json!({ + "tenant": TENANT, + "specs_dir": fixture(fixture_name), + "merge": false, + }); + let response = build_router(state.clone()) + .oneshot( + Request::post("/api/specs/load-dir") + .header("Content-Type", "application/json") + .body(Body::from(body.to_string())) + .expect("build load-dir request"), + ) + .await + .expect("call load-dir"); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 10 * 1024 * 1024) + .await + .expect("consume load-dir verification stream"); + std::str::from_utf8(&body) + .expect("load-dir stream must be UTF-8") + .lines() + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_str(line).expect("load-dir line must be JSON")) + .collect() +} + +async fn load_dir(state: &ServerState, fixture_name: &str) { + let lines = load_dir_lines(state, fixture_name).await; + assert_eq!( + lines.last().and_then(|line| line["all_passed"].as_bool()), + Some(true), + "fixture load must verify and publish: {lines:?}" + ); +} + +fn envelope(actor_id: &str) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: 0, + event_type: "Created".to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: actor_id.to_string(), + }, + } +} + +#[tokio::test(flavor = "current_thread")] +async fn sim_load_dir_restart_tombstones_durable_only_omissions_and_readds() { + let (_guard, _clock, _ids) = install_deterministic_context(216); + let store = SimEventStore::no_faults(216); + let first = sim_state(&store, "arn216-sim-first"); + load_dir(&first, "full_v1").await; + assert_eq!( + store + .spec_declaration_entity_types(TENANT) + .await + .expect("present declarations"), + vec!["Item".to_string(), "Note".to_string()] + ); + + drop(first); + let restarted = sim_state(&store, "arn216-sim-restarted"); + load_dir(&restarted, "item_only").await; + assert_eq!( + store + .spec_declaration_entity_types(TENANT) + .await + .expect("post-replacement declarations"), + vec!["Item".to_string()], + "the restarted registry must tombstone durable-only Note authority" + ); + + let stale_v1 = temper_store_turso::spec_content_hash(NOTE_V1); + let stale = store + .append_with_index_rows( + &format!("{TENANT}:Note:stale-v1"), + 0, + &[envelope("stale-v1")], + &[], + &[], + false, + Some(&stale_v1), + ) + .await + .expect_err("omitted Note writer must be fenced"); + assert!( + stale + .to_string() + .contains("stale live vector declaration fingerprint") + ); + + load_dir(&restarted, "full_v2").await; + let fingerprint_v2 = temper_store_turso::spec_content_hash(NOTE_V2); + store + .append_with_index_rows( + &format!("{TENANT}:Note:current-v2"), + 0, + &[envelope("current-v2")], + &[], + &[], + false, + Some(&fingerprint_v2), + ) + .await + .expect("re-added Note v2 writer"); + assert!( + store + .append_with_index_rows( + &format!("{TENANT}:Note:stale-after-readd"), + 0, + &[envelope("stale-after-readd")], + &[], + &[], + false, + Some(&stale_v1), + ) + .await + .is_err(), + "identical type re-add with changed source must retain monotonic authority" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn turso_load_dir_commits_scoped_replacement_across_restart() { + let db_path = std::env::temp_dir().join(format!( + "temper-arn216-load-dir-{}.db", + uuid::Uuid::new_v4() + )); + let url = format!("file:{}", db_path.display()); + let first_store = TursoEventStore::new(&url, None).await.expect("open Turso"); + let first = turso_state(&first_store, "arn216-turso-first"); + load_dir(&first, "full_v1").await; + drop(first); + drop(first_store); + + let second_store = TursoEventStore::new(&url, None) + .await + .expect("reopen Turso"); + let second = turso_state(&second_store, "arn216-turso-second"); + load_dir(&second, "item_only").await; + drop(second); + drop(second_store); + + let third_store = TursoEventStore::new(&url, None) + .await + .expect("reopen replaced catalog"); + let specs = third_store + .load_specs() + .await + .expect("load committed specs"); + let types = specs + .iter() + .filter(|row| row.tenant == TENANT) + .map(|row| row.entity_type.as_str()) + .collect::>(); + assert_eq!(types, vec!["Item"]); + assert!( + specs + .iter() + .filter(|row| row.tenant == TENANT) + .all(|row| row.committed) + ); + + let third = turso_state(&third_store, "arn216-turso-third"); + load_dir(&third, "full_v2").await; + drop(third); + drop(third_store); + let final_store = TursoEventStore::new(&url, None) + .await + .expect("reopen re-added catalog"); + let note = final_store + .load_specs() + .await + .expect("load re-added specs") + .into_iter() + .find(|row| row.tenant == TENANT && row.entity_type == "Note") + .expect("committed Note v2"); + assert!(note.committed); + assert_eq!( + note.content_hash.as_deref(), + Some(temper_store_turso::spec_content_hash(NOTE_V2).as_str()) + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn failed_verification_preserves_last_committed_catalog_and_registry() { + let db_path = std::env::temp_dir().join(format!( + "temper-arn216-failed-verification-{}.db", + uuid::Uuid::new_v4() + )); + let url = format!("file:{}", db_path.display()); + let store = TursoEventStore::new(&url, None).await.expect("open Turso"); + let mut state = turso_state(&store, "arn216-failed-verification"); + load_dir(&state, "full_v1").await; + + state.verify_subprocess_bin = Some(std::sync::Arc::new( + Path::new("/definitely/missing/temper-verifier").to_path_buf(), + )); + let lines = load_dir_lines(&state, "full_v2").await; + assert_eq!( + lines.last().and_then(|line| line["all_passed"].as_bool()), + Some(false), + "failed verifier must fail the publication stream" + ); + + let committed_note = store + .load_specs() + .await + .expect("load committed catalog") + .into_iter() + .find(|row| row.tenant == TENANT && row.entity_type == "Note") + .expect("last committed Note"); + assert_eq!( + committed_note.content_hash.as_deref(), + Some(temper_store_turso::spec_content_hash(NOTE_V1).as_str()), + "failed verification must preserve the last committed Note bytes" + ); + + let tenant = TenantId::from(TENANT); + let note = state + .get_or_spawn_tenant_actor(&tenant, "Note", "still-v1") + .expect("spawn Note after failed replacement"); + let review = note + .ask::( + EntityMsg::Action { + name: "Review".to_string(), + params: serde_json::json!({"Body": "must remain unavailable"}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: None, + }, + Duration::from_secs(1), + ) + .await + .expect("actor response"); + assert!( + !review.success, + "failed verification must not publish the v2-only Review action" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn existing_actor_hot_swaps_in_place_and_removed_actor_stops() { + let db_path = std::env::temp_dir().join(format!( + "temper-arn216-publication-{}.db", + uuid::Uuid::new_v4() + )); + let url = format!("file:{}", db_path.display()); + let store = TursoEventStore::new(&url, None).await.expect("open Turso"); + let state = turso_state(&store, "arn216-publication-gap"); + load_dir(&state, "full_v1").await; + let tenant = TenantId::from(TENANT); + let existing_actor = state + .get_or_spawn_tenant_actor(&tenant, "Note", "existing-note") + .expect("spawn Note v1 before durable replacement"); + existing_actor + .ask::(EntityMsg::GetState, Duration::from_secs(1)) + .await + .expect("pre-existing actor must finish v1 startup"); + + load_dir(&state, "full_v2").await; + existing_actor + .ask::( + EntityMsg::Action { + name: "Review".to_string(), + params: serde_json::json!({"Body": "survives hot swap"}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: None, + }, + Duration::from_secs(1), + ) + .await + .expect("pre-existing actor must survive and execute Note v2 Review"); + + load_dir(&state, "item_only").await; + assert!( + existing_actor + .ask::(EntityMsg::GetState, Duration::from_millis(100)) + .await + .is_err(), + "an actor whose type is omitted must be stopped" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn publication_snapshot_evicts_unready_restarted_and_same_key_replacements() { + let db_path = std::env::temp_dir().join(format!( + "temper-arn216-actor-identity-{}.db", + uuid::Uuid::new_v4() + )); + let url = format!("file:{}", db_path.display()); + let store = TursoEventStore::new(&url, None).await.expect("open Turso"); + let state = turso_state(&store, "arn216-actor-identity"); + load_dir(&state, "full_v1").await; + let tenant = TenantId::from(TENANT); + let note_types = vec!["Note".to_string()]; + + let unready = state + .get_or_spawn_tenant_actor(&tenant, "Note", "unready-note") + .expect("spawn actor without yielding to pre_start"); + let empty_snapshot = state.ready_actor_identities_for_types(&tenant, ¬e_types); + assert!( + empty_snapshot.is_empty(), + "an ActorRef inserted before pre_start must not be preserved" + ); + state.evict_type_actors_except(&tenant, ¬e_types, &empty_snapshot); + assert!( + unready + .ask::(EntityMsg::GetState, Duration::from_millis(100)) + .await + .is_err(), + "an unready publication-gap actor must be evicted" + ); + + let original = state + .get_or_spawn_tenant_actor(&tenant, "Note", "same-key") + .expect("spawn original same-key actor"); + original + .ask::(EntityMsg::GetState, Duration::from_secs(1)) + .await + .expect("original actor must become ready"); + let original_snapshot = state.ready_actor_identities_for_types(&tenant, ¬e_types); + let original_incarnation = original + .ready_incarnation() + .expect("ready actor must expose its supervised incarnation"); + assert_eq!( + original_snapshot.get(&format!("{TENANT}:Note:same-key")), + Some(&(original.id().uid, original_incarnation)) + ); + + original + .signal(temper_runtime::actor::SystemSignal::Restart) + .expect("request supervised restart"); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if original + .ready_incarnation() + .is_some_and(|incarnation| incarnation != original_incarnation) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("actor must complete a new supervised incarnation"); + assert_eq!( + original.id().uid, + original_snapshot[&format!("{TENANT}:Note:same-key")].0 + ); + state.evict_type_actors_except(&tenant, ¬e_types, &original_snapshot); + assert!( + original + .ask::(EntityMsg::GetState, Duration::from_millis(100)) + .await + .is_err(), + "a supervised restart must not inherit preservation from its prior incarnation" + ); + + let replacement_key = "same-key-replacement"; + let original = state + .get_or_spawn_tenant_actor(&tenant, "Note", replacement_key) + .expect("spawn original actor for same-key replacement"); + original + .ask::(EntityMsg::GetState, Duration::from_secs(1)) + .await + .expect("original replacement-test actor must become ready"); + let original_snapshot = state.ready_actor_identities_for_types(&tenant, ¬e_types); + state.stop_and_remove_entity(&tenant, "Note", replacement_key); + let replacement = state + .get_or_spawn_tenant_actor(&tenant, "Note", replacement_key) + .expect("spawn same-key replacement"); + assert_ne!(replacement.id().uid, original.id().uid); + state.evict_type_actors_except(&tenant, ¬e_types, &original_snapshot); + assert!( + replacement + .ask::(EntityMsg::GetState, Duration::from_millis(100)) + .await + .is_err(), + "a same-key actor with a different uid must not inherit preservation" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn publication_rechecks_supervised_restart_after_initial_eviction() { + let db_path = std::env::temp_dir().join(format!( + "temper-arn216-post-eviction-restart-{}.db", + uuid::Uuid::new_v4() + )); + let url = format!("file:{}", db_path.display()); + let store = TursoEventStore::new(&url, None).await.expect("open Turso"); + let state = turso_state(&store, "arn216-post-eviction-restart"); + load_dir(&state, "full_v1").await; + let tenant = TenantId::from(TENANT); + let note_types = vec!["Note".to_string()]; + let original = state + .get_or_spawn_tenant_actor(&tenant, "Note", "restart-window") + .expect("spawn original actor"); + original + .ask::(EntityMsg::GetState, Duration::from_secs(1)) + .await + .expect("original actor must become ready"); + let preserved = state.ready_actor_identities_for_types(&tenant, ¬e_types); + let original_incarnation = original + .ready_incarnation() + .expect("original incarnation must be ready"); + + state.evict_type_actors_except(&tenant, ¬e_types, &preserved); + original + .signal(temper_runtime::actor::SystemSignal::Restart) + .expect("restart between initial eviction and registry swap"); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if original + .ready_incarnation() + .is_some_and(|incarnation| incarnation != original_incarnation) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("restart must complete before registry publication"); + + state + .registry + .write() + .expect("registry lock") + .try_register_tenant( + TENANT, + temper_spec::csdl::parse_csdl(CSDL_V2).expect("parse v2 CSDL"), + CSDL_V2.to_string(), + &[("Note", NOTE_V2)], + ) + .expect("publish Note v2"); + state.revalidate_type_actors_after_publication(&tenant, ¬e_types, &preserved); + + assert!( + original + .ask::(EntityMsg::GetState, Duration::from_millis(100)) + .await + .is_err(), + "an actor restarted after initial eviction must not survive the registry swap with its old cloned table" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn first_registry_publication_evicts_legacy_fallback_actor() { + let (_guard, _clock, _ids) = install_deterministic_context(219); + let store = SimEventStore::no_faults(219); + let mut state = sim_state(&store, "arn216-first-publication"); + state.transition_tables = std::sync::Arc::new(BTreeMap::from([( + "Note".to_string(), + std::sync::Arc::new(temper_jit::table::TransitionTable::from_ioa_source(NOTE_V1)), + )])); + let tenant = TenantId::from(TENANT); + let legacy = state + .get_or_spawn_tenant_actor(&tenant, "Note", "legacy-note") + .expect("legacy fallback must govern Note before first publication"); + legacy + .ask::(EntityMsg::GetState, Duration::from_secs(1)) + .await + .expect("legacy actor must become ready"); + + load_dir(&state, "full_v2").await; + assert!( + legacy + .ask::(EntityMsg::GetState, Duration::from_millis(100)) + .await + .is_err(), + "first registry publication must evict actors holding cloned fallback tables" + ); + state + .get_or_spawn_tenant_actor(&tenant, "Note", "legacy-note") + .expect("published Note v2 must spawn a registry-backed actor") + .ask::(EntityMsg::GetState, Duration::from_secs(1)) + .await + .expect("registry-backed replacement must be live"); +} diff --git a/crates/temper-server/src/observe/mod.rs b/crates/temper-server/src/observe/mod.rs index fbd3e3cfd..1d08b55f0 100644 --- a/crates/temper-server/src/observe/mod.rs +++ b/crates/temper-server/src/observe/mod.rs @@ -231,3 +231,7 @@ pub fn build_observe_router() -> Router { #[cfg(test)] #[path = "mod_test.rs"] mod tests; + +#[cfg(all(test, feature = "sim"))] +#[path = "load_dir_reconciliation_test.rs"] +mod load_dir_reconciliation_tests; diff --git a/crates/temper-server/src/observe/mod_test.rs b/crates/temper-server/src/observe/mod_test.rs index b5bf6abd0..a260c43ce 100644 --- a/crates/temper-server/src/observe/mod_test.rs +++ b/crates/temper-server/src/observe/mod_test.rs @@ -909,6 +909,16 @@ async fn test_load_inline_supports_nested_paths() { .unwrap(); assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 10 * 1024 * 1024) + .await + .expect("consume inline verification stream"); + let summary = std::str::from_utf8(&body) + .expect("inline verification stream must be UTF-8") + .lines() + .rfind(|line| !line.is_empty()) + .map(|line| serde_json::from_str::(line).expect("summary JSON")) + .expect("inline verification summary"); + assert_eq!(summary["all_passed"], true); let registry = state.registry.read().unwrap(); let tenant = TenantId::new("nested-inline"); diff --git a/crates/temper-server/src/observe/specs/load_dir.rs b/crates/temper-server/src/observe/specs/load_dir.rs index ca3500398..1bc99ba45 100644 --- a/crates/temper-server/src/observe/specs/load_dir.rs +++ b/crates/temper-server/src/observe/specs/load_dir.rs @@ -1,6 +1,7 @@ use axum::extract::State; use axum::http::StatusCode; use axum::response::Json; +use temper_runtime::tenant::TenantId; use temper_spec::automaton::LintSeverity; use temper_spec::cross_invariant::{ CrossInvariantLintSeverity, lint_cross_invariants, parse_cross_invariants, @@ -13,8 +14,17 @@ use super::super::specs_helpers::{ }; use super::types::LoadDirRequest; use super::verification_stream::build_verification_stream_response; +use crate::registry::{EntityVerificationResult, VerificationStatus}; use crate::state::ServerState; +pub(super) struct PendingCatalogPublication { + csdl: temper_spec::csdl::CsdlDocument, + csdl_xml: String, + specs_dir: String, + merge: bool, + cross_invariants_toml: Option, +} + /// POST /api/specs/load-dir -- hot-load specs from a directory into the running server./// /// Reads CSDL and IOA files from `specs_dir`, registers them under `tenant`, /// emits design-time SSE events for each entity, and spawns background @@ -184,90 +194,167 @@ pub(crate) async fn handle_load_dir( return build_ndjson_response(StatusCode::BAD_REQUEST, lines); } - // Persist loaded specs first when Postgres is configured. - let csdl_xml_for_db = csdl_xml.clone(); - for (entity_type, ioa_source) in &ioa_sources { - state - .upsert_spec_source(&body.tenant, entity_type, ioa_source, &csdl_xml_for_db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?; - } + // Verification owns only staged bytes. Committed authority and the live + // registry remain unchanged until the entire exact catalog passes. state - .upsert_tenant_constraints(&body.tenant, cross_invariants_toml.as_deref()) + .stage_spec_catalog_update(&body.tenant, &ioa_sources, &csdl_xml) .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?; - // Register into shared registry after persistence succeeds. - let ioa_pairs: Vec<(&str, &str)> = ioa_sources - .iter() - .map(|(k, v)| (k.as_str(), v.as_str())) + // Stream NDJSON response: verification runs inline and results are streamed per-entity. + // Any agent calling this endpoint gets verification results without polling. + let lint_warning_lines: Vec = lint_findings + .into_iter() + .filter(|f| matches!(f.severity, LintSeverity::Warning)) + .map(|f| lint_ndjson_line(&f)) + .collect(); + let cross_lint_warning_lines: Vec = cross_lint_findings + .into_iter() + .filter(|f| matches!(f.severity, CrossInvariantLintSeverity::Warning)) + .map(|f| cross_lint_ndjson_line(&f)) .collect(); + Ok(build_verification_stream_response( + state, + body.tenant, + entity_names, + ioa_sources, + lint_warning_lines, + cross_lint_warning_lines, + PendingCatalogPublication { + csdl, + csdl_xml, + specs_dir: body.specs_dir, + merge: body.merge, + cross_invariants_toml, + }, + )) +} + +pub(super) async fn finalize_verified_load( + state: &ServerState, + tenant: &str, + ioa_sources: &std::collections::BTreeMap, + verification_results: &std::collections::BTreeMap, + pending: PendingCatalogPublication, +) -> Result<(), String> { + let _catalog_update_guard = state.spec_catalog_update_lock.lock().await; + let tenant_id = TenantId::from(tenant); + let incoming_entity_types = ioa_sources.keys().cloned().collect::>(); + let incoming = ioa_sources + .keys() + .map(String::as_str) + .collect::>(); + let (had_registry_config, additional_removed_entity_types) = { + let registry = state + .registry + .read() + .map_err(|error| format!("registry lock poisoned: {error}"))?; + let had_registry_config = registry.get_tenant(&tenant_id).is_some(); + let mut existing = registry + .entity_types(&tenant_id) + .into_iter() + .map(str::to_string) + .collect::>(); + if !had_registry_config { + existing.extend(state.transition_tables.keys().cloned()); + } + let removed = if pending.merge { + Vec::new() + } else { + existing + .into_iter() + .filter(|entity_type| !incoming.contains(entity_type.as_str())) + .collect() + }; + (had_registry_config, removed) + }; + let preserved_incoming_actors = if had_registry_config { + state.ready_actor_identities_for_types(&tenant_id, &incoming_entity_types) + } else { + std::collections::BTreeMap::new() + }; + let removed_entity_types = state + .persist_verified_spec_catalog_update( + tenant, + ioa_sources, + &pending.csdl_xml, + &additional_removed_entity_types, + !pending.merge, + pending.cross_invariants_toml.as_deref(), + ) + .await?; + let replaced_entity_types = removed_entity_types + .iter() + .cloned() + .chain(incoming_entity_types.iter().cloned()) + .collect::>() + .into_iter() + .collect::>(); + let actor_publication_guard = state + .actor_spec_publication_lock + .write() + .map_err(|error| format!("actor/spec publication lock poisoned: {error}"))?; + state.evict_type_actors_except( + &tenant_id, + &replaced_entity_types, + &preserved_incoming_actors, + ); + let ioa_pairs = ioa_sources + .iter() + .map(|(entity_type, source)| (entity_type.as_str(), source.as_str())) + .collect::>(); { - let mut registry = state.registry.write().unwrap(); // ci-ok: infallible lock + let mut registry = state + .registry + .write() + .map_err(|error| format!("registry lock poisoned: {error}"))?; registry .try_register_tenant_with_reactions_and_constraints( - body.tenant.as_str(), - csdl, - csdl_xml, + tenant, + pending.csdl, + pending.csdl_xml, &ioa_pairs, Vec::new(), - cross_invariants_toml.clone(), - body.merge, + pending.cross_invariants_toml, + pending.merge, ) - .map_err(|e| { - ( - StatusCode::BAD_REQUEST, - format!("Failed to register specs: {e}"), - ) - })?; + .map_err(|error| format!("failed to register verified specs: {error}"))?; + for (entity_type, result) in verification_results { + registry.set_verification_status( + &tenant_id, + entity_type, + VerificationStatus::Completed(result.clone()), + ); + } } + state.revalidate_type_actors_after_publication( + &tenant_id, + &replaced_entity_types, + &preserved_incoming_actors, + ); + drop(actor_publication_guard); state.rebuild_reaction_dispatcher(); + state.populate_vector_index_from_snapshots(&tenant_id).await; if !state.data_dir.as_os_str().is_empty() { let registry_path = state.data_dir.join("specs-registry.json"); let mut specs_registry = std::collections::BTreeMap::::new(); - - if let Ok(content) = std::fs::read_to_string(®istry_path) { - // determinism-ok: HTTP handler reads specs registry - if let Ok(value) = serde_json::from_str::(&content) - && let Some(obj) = value.as_object() - { - for (tenant, specs_dir) in obj { - if let Some(specs_dir) = specs_dir.as_str() { - specs_registry.insert(tenant.clone(), specs_dir.to_string()); - } + if let Ok(content) = std::fs::read_to_string(®istry_path) + && let Ok(value) = serde_json::from_str::(&content) + && let Some(obj) = value.as_object() + { + for (existing_tenant, specs_dir) in obj { + if let Some(specs_dir) = specs_dir.as_str() { + specs_registry.insert(existing_tenant.clone(), specs_dir.to_string()); } } } - - specs_registry.insert(body.tenant.clone(), body.specs_dir.clone()); - + specs_registry.insert(tenant.to_string(), pending.specs_dir); if let Ok(encoded) = serde_json::to_string_pretty(&specs_registry) { let _ = std::fs::create_dir_all(&state.data_dir); // determinism-ok: HTTP handler creates data dir let _ = std::fs::write(registry_path, encoded); // determinism-ok: HTTP handler writes specs registry } } - - // Stream NDJSON response: verification runs inline and results are streamed per-entity. - // Any agent calling this endpoint gets verification results without polling. - let lint_warning_lines: Vec = lint_findings - .into_iter() - .filter(|f| matches!(f.severity, LintSeverity::Warning)) - .map(|f| lint_ndjson_line(&f)) - .collect(); - let cross_lint_warning_lines: Vec = cross_lint_findings - .into_iter() - .filter(|f| matches!(f.severity, CrossInvariantLintSeverity::Warning)) - .map(|f| cross_lint_ndjson_line(&f)) - .collect(); - - Ok(build_verification_stream_response( - state, - body.tenant, - entity_names, - ioa_sources, - lint_warning_lines, - cross_lint_warning_lines, - )) + Ok(()) } diff --git a/crates/temper-server/src/observe/specs/verification_stream.rs b/crates/temper-server/src/observe/specs/verification_stream.rs index 7b2156a59..c22e78dd2 100644 --- a/crates/temper-server/src/observe/specs/verification_stream.rs +++ b/crates/temper-server/src/observe/specs/verification_stream.rs @@ -5,7 +5,7 @@ use tokio_stream::wrappers::ReceiverStream; use temper_runtime::scheduler::sim_now; -use crate::registry::VerificationStatus; +use super::load_dir::{PendingCatalogPublication, finalize_verified_load}; use crate::state::ServerState; pub(super) fn build_verification_stream_response( @@ -15,6 +15,7 @@ pub(super) fn build_verification_stream_response( ioa_sources: BTreeMap, lint_warning_lines: Vec, cross_lint_warning_lines: Vec, + pending_publication: PendingCatalogPublication, ) -> axum::response::Response { let (tx, rx) = tokio::sync::mpsc::channel::>(100); let state_for_task = state.clone(); @@ -51,6 +52,7 @@ pub(super) fn build_verification_stream_response( let mut entity_results: std::collections::BTreeMap = std::collections::BTreeMap::new(); + let mut verification_results = BTreeMap::new(); for entity_name in &entity_names { // Emit design-time events for UI (spec_loaded + verify_started) @@ -79,32 +81,6 @@ pub(super) fn build_verification_stream_response( entity_results.insert(entity_name.clone(), false); continue; } - if let Err(e) = state_for_task - .persist_spec_verification(&tenant, entity_name, "running", None) - .await - { - tracing::error!(tenant = %tenant, entity = %entity_name, error = %e, "failed to persist running verification status"); - let _ = tx - .send(Ok(serde_json::to_string(&serde_json::json!({ - "type": "verification_error", - "entity": entity_name, - "error": e, - })) - .unwrap() // ci-ok: infallible serialization - + "\n")) - .await; - entity_results.insert(entity_name.clone(), false); - continue; - } - { - let mut registry = state_for_task.registry.write().unwrap(); // ci-ok: infallible lock - registry.set_verification_status( - &tenant.clone().into(), - entity_name, - VerificationStatus::Running, - ); - } - let started_event = crate::state::DesignTimeEvent { kind: "verify_started".to_string(), entity_type: entity_name.clone(), @@ -320,43 +296,7 @@ pub(super) fn build_verification_stream_response( .await; entity_results.insert(entity_name.clone(), cascade_result.all_passed); - - let passed_count = entity_result.levels.iter().filter(|l| l.passed).count(); - let final_status = if entity_result.all_passed { - "passed" - } else if passed_count == 0 { - "failed" - } else { - "partial" - }; - if let Err(e) = state_for_task - .persist_spec_verification( - &tenant, - entity_name, - final_status, - Some(&entity_result), - ) - .await - { - tracing::error!(tenant = %tenant, entity = %entity_name, error = %e, "failed to persist completed verification status"); - let _ = tx - .send(Ok(serde_json::to_string(&serde_json::json!({ - "type": "verification_error", - "entity": entity_name, - "error": e, - })) - .unwrap() // ci-ok: infallible serialization - + "\n")) - .await; - continue; - } - if let Ok(mut reg) = state_for_task.registry.write() { - reg.set_verification_status( - &tenant.clone().into(), - entity_name, - VerificationStatus::Completed(entity_result.clone()), - ); - } + verification_results.insert(entity_name.clone(), entity_result); let done_event = crate::state::DesignTimeEvent { kind: "verify_done".to_string(), entity_type: entity_name.clone(), @@ -387,44 +327,6 @@ pub(super) fn build_verification_stream_response( } Err(e) => { entity_results.insert(entity_name.clone(), false); - let failure_result = crate::registry::EntityVerificationResult { - all_passed: false, - levels: vec![crate::registry::EntityLevelSummary { - level: "VerificationTask".to_string(), - passed: false, - summary: format!("Verification failed for {entity_name}: {e}"), - details: None, - }], - verified_at: sim_now().to_rfc3339(), - }; - if let Err(persist_err) = state_for_task - .persist_spec_verification( - &tenant, - entity_name, - "failed", - Some(&failure_result), - ) - .await - { - tracing::error!(tenant = %tenant, entity = %entity_name, error = %persist_err, "failed to persist failed verification status"); - let _ = tx - .send(Ok(serde_json::to_string(&serde_json::json!({ - "type": "verification_error", - "entity": entity_name, - "error": persist_err, - })) - .unwrap() // ci-ok: infallible serialization - + "\n")) - .await; - continue; - } - if let Ok(mut reg) = state_for_task.registry.write() { - reg.set_verification_status( - &tenant.clone().into(), - entity_name, - VerificationStatus::Completed(failure_result.clone()), - ); - } let fail_event = crate::state::DesignTimeEvent { kind: "verify_done".to_string(), entity_type: entity_name.clone(), @@ -455,12 +357,39 @@ pub(super) fn build_verification_stream_response( } // Stream final summary - let all_passed = entity_results.values().all(|&p| p); + let verification_passed = entity_results.len() == entity_names.len() + && entity_results.values().all(|&passed| passed); + let publication_result = if verification_passed { + finalize_verified_load( + &state_for_task, + &tenant, + &ioa_sources, + &verification_results, + pending_publication, + ) + .await + } else { + Ok(()) + }; + if let Err(error) = &publication_result { + tracing::error!(tenant = %tenant, error = %error, "failed to publish verified catalog"); + let _ = tx + .send(Ok(serde_json::to_string(&serde_json::json!({ + "type": "publication_error", + "tenant": &tenant, + "error": error, + })) + .unwrap() // ci-ok: infallible serialization + + "\n")) + .await; + } + let all_passed = verification_passed && publication_result.is_ok(); let _ = tx .send(Ok(serde_json::to_string(&serde_json::json!({ "type": "summary", "tenant": &tenant, "all_passed": all_passed, + "published": all_passed, "entities": entity_results, })) .unwrap() // ci-ok: infallible serialization diff --git a/crates/temper-server/src/platform_store.rs b/crates/temper-server/src/platform_store.rs index 6fb57ed66..f747dcb4f 100644 --- a/crates/temper-server/src/platform_store.rs +++ b/crates/temper-server/src/platform_store.rs @@ -44,6 +44,17 @@ pub struct SpecVerificationUpdate<'a> { pub verification_result_json: Option<&'a str>, } +/// Exact staged spec bytes owned by one atomic catalog publication. +#[derive(Debug, Clone, Copy)] +pub struct SpecCommitExpectation<'a> { + /// Entity type whose staged bytes may be promoted. + pub entity_type: &'a str, + /// Expected IOA content hash. + pub content_hash: &'a str, + /// Expected CSDL bytes. + pub csdl_xml: &'a str, +} + /// WASM module row returned by [`PlatformStore`] WASM queries. #[derive(Debug, Clone)] pub struct WasmModuleRow { @@ -130,6 +141,21 @@ pub trait PlatformStore: Send + Sync { /// Mark all uncommitted specs for a tenant as committed. async fn commit_specs(&self, tenant: &str) -> Result<(), String>; + /// Atomically promote only the exact staged specs owned by one operation. + async fn commit_spec_batch( + &self, + tenant: &str, + expected: &[SpecCommitExpectation<'_>], + ) -> Result<(), String>; + /// Atomically persist verification and commit only the expected spec bytes. + async fn commit_verified_spec( + &self, + tenant: &str, + entity_type: &str, + expected_content_hash: &str, + expected_csdl_xml: &str, + update: SpecVerificationUpdate<'_>, + ) -> Result<(), String>; /// Delete all uncommitted specs across all tenants. async fn delete_uncommitted_specs(&self) -> Result; @@ -275,6 +301,43 @@ impl PlatformStore for TursoEventStore { async fn commit_specs(&self, tenant: &str) -> Result<(), String> { self.commit_specs(tenant).await.map_err(|e| e.to_string()) } + async fn commit_spec_batch( + &self, + tenant: &str, + expected: &[SpecCommitExpectation<'_>], + ) -> Result<(), String> { + let expected = expected + .iter() + .map(|spec| (spec.entity_type, spec.content_hash, spec.csdl_xml)) + .collect::>(); + self.commit_spec_batch(tenant, &expected) + .await + .map_err(|e| e.to_string()) + } + async fn commit_verified_spec( + &self, + tenant: &str, + entity_type: &str, + expected_content_hash: &str, + expected_csdl_xml: &str, + update: SpecVerificationUpdate<'_>, + ) -> Result<(), String> { + self.commit_verified_spec( + tenant, + entity_type, + expected_content_hash, + expected_csdl_xml, + TursoSpecVerificationUpdate { + status: update.status, + verified: update.verified, + levels_passed: update.levels_passed, + levels_total: update.levels_total, + verification_result_json: update.verification_result_json, + }, + ) + .await + .map_err(|e| e.to_string()) + } async fn delete_uncommitted_specs(&self) -> Result { self.delete_uncommitted_specs() .await @@ -531,6 +594,44 @@ impl PlatformStore for PostgresEventStore { async fn commit_specs(&self, tenant: &str) -> Result<(), String> { self.commit_specs(tenant).await.map_err(|e| e.to_string()) } + async fn commit_spec_batch( + &self, + tenant: &str, + expected: &[SpecCommitExpectation<'_>], + ) -> Result<(), String> { + let expected = expected + .iter() + .map(|spec| (spec.entity_type, spec.content_hash, spec.csdl_xml)) + .collect::>(); + self.commit_spec_batch(tenant, &expected) + .await + .map_err(|e| e.to_string()) + } + + async fn commit_verified_spec( + &self, + tenant: &str, + entity_type: &str, + expected_content_hash: &str, + expected_csdl_xml: &str, + update: SpecVerificationUpdate<'_>, + ) -> Result<(), String> { + self.commit_verified_spec( + tenant, + entity_type, + expected_content_hash, + expected_csdl_xml, + PostgresSpecVerificationUpdate { + status: update.status, + verified: update.verified, + levels_passed: update.levels_passed, + levels_total: update.levels_total, + verification_result_json: update.verification_result_json, + }, + ) + .await + .map_err(|e| e.to_string()) + } async fn delete_uncommitted_specs(&self) -> Result { self.delete_uncommitted_specs() @@ -853,6 +954,8 @@ mod sim_platform_store { faults: SimPlatformFaultConfig, /// Specs keyed by (tenant, entity_type). specs: BTreeMap<(String, String), SpecRow>, + /// Replacement bytes awaiting verification, keyed by tenant/type. + staged_specs: BTreeMap<(String, String), SpecRow>, /// Verification cache: (tenant, entity_type) -> (content_hash, verified). verification_cache: BTreeMap<(String, String), (String, bool)>, /// Cedar policies keyed by tenant. @@ -879,6 +982,7 @@ mod sim_platform_store { rng: DeterministicRng::new(seed), faults, specs: BTreeMap::new(), + staged_specs: BTreeMap::new(), verification_cache: BTreeMap::new(), policies: BTreeMap::new(), policy_entries: BTreeMap::new(), @@ -965,7 +1069,7 @@ mod sim_platform_store { } let key = (tenant.to_string(), entity_type.to_string()); - inner.specs.insert( + inner.staged_specs.insert( key, SpecRow { tenant: tenant.to_string(), @@ -1004,24 +1108,122 @@ mod sim_platform_store { inner .specs .remove(&(tenant.to_string(), entity_type.to_string())); + inner + .staged_specs + .remove(&(tenant.to_string(), entity_type.to_string())); Ok(()) } async fn commit_specs(&self, tenant: &str) -> Result<(), String> { let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - for spec in inner.specs.values_mut() { - if spec.tenant == tenant { - spec.committed = true; + let keys = inner + .staged_specs + .keys() + .filter(|(candidate_tenant, _)| candidate_tenant == tenant) + .cloned() + .collect::>(); + for key in keys { + let mut spec = inner + .staged_specs + .remove(&key) + .expect("collected staged spec key must exist"); // ci-ok: same-lock key snapshot + spec.committed = true; + inner.specs.insert(key, spec); + } + Ok(()) + } + + async fn commit_spec_batch( + &self, + tenant: &str, + expected: &[SpecCommitExpectation<'_>], + ) -> Result<(), String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + let failure_probability = inner.faults.spec_write_failure_prob; + if inner.rng.chance(failure_probability) { + return Err("SimPlatformStore: injected spec batch commit failure".into()); + } + let mut entity_types = BTreeSet::new(); + for spec in expected { + if !entity_types.insert(spec.entity_type) { + return Err(format!( + "duplicate spec batch entity type {tenant}/{}", + spec.entity_type + )); + } + let key = (tenant.to_string(), spec.entity_type.to_string()); + let staged = inner + .staged_specs + .get(&key) + .ok_or_else(|| format!("missing staged spec {tenant}/{}", spec.entity_type))?; + if staged.content_hash != spec.content_hash + || staged.csdl_xml.as_deref() != Some(spec.csdl_xml) + { + return Err(format!( + "staged spec fingerprint changed for {tenant}/{}", + spec.entity_type + )); + } + } + for spec in expected { + let key = (tenant.to_string(), spec.entity_type.to_string()); + let mut staged = inner + .staged_specs + .remove(&key) + .expect("validated staged spec must still exist"); // ci-ok: same-lock validation + staged.committed = true; + inner.specs.insert(key.clone(), staged); + inner + .verification_cache + .insert(key, (spec.content_hash.to_string(), false)); + } + Ok(()) + } + + async fn commit_verified_spec( + &self, + tenant: &str, + entity_type: &str, + expected_content_hash: &str, + expected_csdl_xml: &str, + update: SpecVerificationUpdate<'_>, + ) -> Result<(), String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + let failure_probability = inner.faults.spec_write_failure_prob; + if inner.rng.chance(failure_probability) { + return Err("SimPlatformStore: injected verified spec commit failure".into()); + } + let key = (tenant.to_string(), entity_type.to_string()); + { + let spec = inner + .staged_specs + .get_mut(&key) + .ok_or_else(|| format!("missing staged spec {tenant}/{entity_type}"))?; + if spec.content_hash != expected_content_hash + || spec.csdl_xml.as_deref() != Some(expected_csdl_xml) + { + return Err(format!( + "staged spec fingerprint changed for {tenant}/{entity_type}" + )); } + spec.committed = true; } + let spec = inner + .staged_specs + .remove(&key) + .expect("verified staged spec must still exist"); // ci-ok: same-lock validation + inner.specs.insert(key.clone(), spec); + inner + .verification_cache + .insert(key, (expected_content_hash.to_string(), update.verified)); Ok(()) } async fn delete_uncommitted_specs(&self) -> Result { let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - let before = inner.specs.len(); - inner.specs.retain(|_, s| s.committed); - Ok(before - inner.specs.len()) + let removed = inner.staged_specs.len(); + inner.staged_specs.clear(); + Ok(removed) } async fn load_verification_cache( @@ -1293,4 +1495,84 @@ mod sim_platform_store { Ok(()) } } + + #[cfg(test)] + mod tests { + use super::*; + + #[tokio::test] + async fn spec_batch_commit_invalidates_same_ioa_verification_and_rejects_duplicates() { + let store = SimPlatformStore::no_faults(216); + let ioa = "[automaton]\nname = \"Item\"\n"; + let hash = "same-ioa-hash"; + let csdl_a = ""; + let csdl_b = ""; + + store + .upsert_spec("t", "Item", ioa, csdl_a, hash) + .await + .expect("stage Item with CSDL A"); + store + .commit_verified_spec( + "t", + "Item", + hash, + csdl_a, + SpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .expect("commit verified Item with CSDL A"); + store + .upsert_spec("t", "Item", ioa, csdl_b, hash) + .await + .expect("stage Item with CSDL B"); + store + .commit_spec_batch( + "t", + &[SpecCommitExpectation { + entity_type: "Item", + content_hash: hash, + csdl_xml: csdl_b, + }], + ) + .await + .expect("commit Item batch with CSDL B"); + assert_eq!( + store + .load_verification_cache("t") + .await + .expect("load Item verification cache") + .get("Item"), + Some(&(hash.to_string(), false)) + ); + + store + .upsert_spec("t", "Issue", ioa, csdl_a, hash) + .await + .expect("stage Issue"); + let duplicate = SpecCommitExpectation { + entity_type: "Issue", + content_hash: hash, + csdl_xml: csdl_a, + }; + store + .commit_spec_batch("t", &[duplicate, duplicate]) + .await + .expect_err("duplicate entity expectations must fail atomically"); + assert!( + store + .load_specs() + .await + .expect("load committed specs") + .iter() + .all(|row| row.entity_type != "Issue") + ); + } + } } diff --git a/crates/temper-server/src/registry/mod.rs b/crates/temper-server/src/registry/mod.rs index 8319e7208..0716d79c9 100644 --- a/crates/temper-server/src/registry/mod.rs +++ b/crates/temper-server/src/registry/mod.rs @@ -180,6 +180,10 @@ impl SpecRegistry { } if let Some(existing_config) = self.tenants.get_mut(&tenant) { + existing_config.revision = existing_config + .revision + .checked_add(1) + .expect("tenant registry revision exhausted"); // Hot-reload path: swap tables on existing entities, add new ones. if merge { // Merge mode: combine incoming CSDL/entity-set-map with existing. @@ -219,7 +223,9 @@ impl SpecRegistry { source: e.to_string(), } })?; - let table = TransitionTable::from_automaton(&automaton); + let mut table = TransitionTable::from_automaton(&automaton); + table.spec_declaration_fingerprint = + Some(temper_store_turso::spec_content_hash(ioa_source)); let integrations = automaton.integrations.clone(); if let Some(existing_spec) = existing_config.entities.get_mut(*entity_type) { @@ -286,7 +292,9 @@ impl SpecRegistry { source: e.to_string(), } })?; - let table = TransitionTable::from_automaton(&automaton); + let mut table = TransitionTable::from_automaton(&automaton); + table.spec_declaration_fingerprint = + Some(temper_store_turso::spec_content_hash(ioa_source)); let integrations = automaton.integrations.clone(); entities.insert( entity_type.to_string(), @@ -308,6 +316,7 @@ impl SpecRegistry { self.tenants.insert( tenant, TenantConfig { + revision: 1, csdl: Arc::new(csdl), csdl_xml: Arc::new(csdl_xml), entity_set_map, @@ -498,6 +507,17 @@ impl SpecRegistry { } } + /// Remove the verification gate for a specific entity type. + /// + /// This is used by legacy compatibility constructors whose supplied specs + /// were historically trusted without running the design-time cascade. + pub fn remove_verification_status(&mut self, tenant: &TenantId, entity_type: &str) -> bool { + self.tenants + .get_mut(tenant) + .and_then(|config| config.verification.remove(entity_type)) + .is_some() + } + /// Get verification status for a specific entity type. pub fn get_verification_status( &self, @@ -644,6 +664,13 @@ mod tests { registry.register_tenant("alpha", csdl, xml, &[("Order", ORDER_IOA)]); + assert_eq!( + registry + .get_tenant(&TenantId::new("alpha")) + .unwrap() + .revision, + 1 + ); let spec = registry.get_spec(&TenantId::new("alpha"), "Order").unwrap(); assert_eq!(spec.automaton.automaton.name, "Order"); assert!(!spec.ioa_source.is_empty()); @@ -850,6 +877,12 @@ assert = 'related(Order, OrderId).status in ["Active"]' ) .expect("replace should succeed"); + assert_eq!( + registry.get_tenant(&tenant).unwrap().revision, + 2, + "tenant declaration revision advances across replacement" + ); + assert!( registry.get_table(&tenant, "Order").is_none(), "Order removed in replace" diff --git a/crates/temper-server/src/registry/types.rs b/crates/temper-server/src/registry/types.rs index ad46887db..a8a6e98a3 100644 --- a/crates/temper-server/src/registry/types.rs +++ b/crates/temper-server/src/registry/types.rs @@ -145,6 +145,9 @@ pub struct RelationGraph { /// A registered tenant with its specs and entity configuration. #[derive(Debug, Clone)] pub struct TenantConfig { + /// Process-local monotonic revision for declaration snapshots. Persistent + /// stores validate the accompanying fingerprint against their spec catalog. + pub revision: u64, /// The CSDL document describing this tenant's entity model. pub csdl: Arc, /// Raw CSDL XML for serving via `$metadata`. diff --git a/crates/temper-server/src/registry_bootstrap.rs b/crates/temper-server/src/registry_bootstrap.rs index 9764fa70e..9def146f5 100644 --- a/crates/temper-server/src/registry_bootstrap.rs +++ b/crates/temper-server/src/registry_bootstrap.rs @@ -208,20 +208,24 @@ fn populate_registry( Ok(restored_specs) } -/// Restore a [`SpecRegistry`] from Postgres. -pub async fn restore_registry_from_postgres( - registry: &mut SpecRegistry, - pool: &sqlx::PgPool, -) -> Result { - let rows: Vec = sqlx::query_as( +async fn load_postgres_spec_rows(pool: &sqlx::PgPool) -> Result, String> { + sqlx::query_as( "SELECT tenant, entity_type, ioa_source, csdl_xml, verification_status, verified, \ levels_passed, levels_total, verification_result, updated_at \ - FROM specs \ + FROM specs WHERE committed = true \ ORDER BY tenant, entity_type", ) .fetch_all(pool) .await - .map_err(|e| format!("Failed to read specs from Postgres: {e}"))?; + .map_err(|e| format!("Failed to read specs from Postgres: {e}")) +} + +/// Restore a [`SpecRegistry`] from Postgres. +pub async fn restore_registry_from_postgres( + registry: &mut SpecRegistry, + pool: &sqlx::PgPool, +) -> Result { + let rows = load_postgres_spec_rows(pool).await?; #[derive(sqlx::FromRow)] struct ConstraintRow { diff --git a/crates/temper-server/src/registry_bootstrap_test.rs b/crates/temper-server/src/registry_bootstrap_test.rs index dc0e73056..7cf93075f 100644 --- a/crates/temper-server/src/registry_bootstrap_test.rs +++ b/crates/temper-server/src/registry_bootstrap_test.rs @@ -191,3 +191,56 @@ fn row_to_registry_status_failed() { other => panic!("Expected Restored, got {other:?}"), } } + +#[tokio::test] +async fn postgres_restore_does_not_publish_uncommitted_staging() { + let Ok(database_url) = std::env::var("DATABASE_URL") else { + return; + }; + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("migrate Postgres"); + let store = temper_store_postgres::PostgresEventStore::new(pool.clone()); + let tenant = format!("registry-staged-{}", uuid::Uuid::new_v4()); + let ioa_a = include_str!("../../../test-fixtures/specs/order.ioa.toml"); + let ioa_b = ioa_a.replace("#", "# staged restart\n#"); + let csdl_xml = csdl_xml_for("Order", "Orders"); + let fingerprint_a = temper_store_turso::spec_content_hash(ioa_a); + let fingerprint_b = temper_store_turso::spec_content_hash(&ioa_b); + + store + .upsert_spec(&tenant, "Order", ioa_a, &csdl_xml, &fingerprint_a) + .await + .expect("stage declaration A"); + store + .commit_specs(&tenant) + .await + .expect("commit declaration A"); + store + .upsert_spec(&tenant, "Order", &ioa_b, &csdl_xml, &fingerprint_b) + .await + .expect("stage declaration B"); + + let authority: (String, bool) = sqlx::query_as( + "SELECT declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Order'", + ) + .bind(&tenant) + .fetch_one(&pool) + .await + .expect("read committed authority"); + assert_eq!(authority, (fingerprint_a, true)); + + let restored_rows = load_postgres_spec_rows(&pool) + .await + .expect("load committed registry rows"); + let restored = restored_rows + .iter() + .find(|row| row.tenant == tenant && row.entity_type == "Order") + .expect("startup must retain committed A while B is staged"); + assert_eq!(restored.ioa_source, ioa_a); +} diff --git a/crates/temper-server/src/state/dispatch/composite.rs b/crates/temper-server/src/state/dispatch/composite.rs index 554a26a59..1efb40672 100644 --- a/crates/temper-server/src/state/dispatch/composite.rs +++ b/crates/temper-server/src/state/dispatch/composite.rs @@ -71,6 +71,7 @@ struct PreflightCompositeTarget { struct AtomicCompositeStream { entity_type: String, entity_id: String, + table: Arc, target_existed: bool, state: EntityState, expected_sequence: u64, @@ -286,7 +287,15 @@ impl crate::state::ServerState { ) .await?; - let table = self.transition_table_for_dispatch(tenant, &write.entity_type)?; + // The first write for a stream fixes the transition-table snapshot for + // the whole atomic batch. A hot swap may affect the next dispatch, but + // it must not relabel events derived from the old table with the new + // declaration fingerprint at commit time. + let table = streams + .get(&persistence_id) + .expect("stream inserted before table lookup") + .table + .clone(); let cross_entity_booleans = if table_has_cross_entity_guards_for_action(&table, &write.action) { self.resolve_cross_entity_guards( @@ -374,15 +383,25 @@ impl crate::state::ServerState { } let stage_ms = stage_started_at.map(|started| started.elapsed().as_millis() as u64); - let appends = streams + let mut appends = Vec::new(); + for (persistence_id, stream) in streams .iter() .filter(|(_, stream)| !stream.events.is_empty()) - .map(|(persistence_id, stream)| PersistenceAppend { + { + let vector_rows = crate::vector_index::rows_for_entity_state( + &stream.table.vectors, + &stream.state.status, + &stream.state.fields, + ); + appends.push(PersistenceAppend { persistence_id: persistence_id.clone(), expected_sequence: stream.expected_sequence, events: stream.events.clone(), - }) - .collect::>(); + vector_rows, + reconcile_vectors: !stream.table.vectors.is_empty(), + spec_declaration_fingerprint: stream.table.spec_declaration_fingerprint.clone(), + }); + } if appends.is_empty() { return Ok(true); } @@ -462,11 +481,11 @@ impl crate::state::ServerState { if streams.contains_key(&persistence_id) { return Ok(()); } + let table = self.transition_table_for_dispatch(tenant, entity_type)?; let (target_exists, mut state) = if let Some(target) = preflight_target { (target.target_existed, target.state.clone()) } else { - let table = self.transition_table_for_dispatch(tenant, entity_type)?; let target_exists = self .ensure_entity_loaded(tenant, entity_type, entity_id) .await; @@ -504,6 +523,7 @@ impl crate::state::ServerState { AtomicCompositeStream { entity_type: entity_type.to_string(), entity_id: entity_id.to_string(), + table, target_existed: target_exists, state, expected_sequence, @@ -903,18 +923,8 @@ impl crate::state::ServerState { tenant: &TenantId, entity_type: &str, ) -> Result, DispatchError> { - if let Some(table) = self - .registry - .read() - .map_err(|e| DispatchError::Internal(format!("registry lock poisoned: {e}")))? - .get_table(tenant, entity_type) - { - return Ok(table); - } - - self.transition_tables - .get(entity_type) - .cloned() + self.transition_table_for_tenant(tenant, entity_type) + .map_err(DispatchError::Internal)? .ok_or_else(|| DispatchError::Ungoverned(entity_type.to_string())) } diff --git a/crates/temper-server/src/state/dispatch/composite_test.rs b/crates/temper-server/src/state/dispatch/composite_test.rs index 24e6c9842..201d78422 100644 --- a/crates/temper-server/src/state/dispatch/composite_test.rs +++ b/crates/temper-server/src/state/dispatch/composite_test.rs @@ -2,6 +2,8 @@ use std::collections::BTreeMap; use serde_json::json; use temper_runtime::ActorSystem; +#[cfg(feature = "sim")] +use temper_runtime::persistence::{EntityVectorRow, EventStore}; use temper_spec::csdl::parse_csdl; #[cfg(feature = "sim")] use temper_store_sim::SimEventStore; @@ -99,6 +101,8 @@ const COMPOSITE_CSDL: &str = r#" + + @@ -223,12 +227,29 @@ name = "Child" states = ["Draft", "Active", "Deleted"] initial = "Draft" +[[state]] +name = "Embedding" +type = "string" +initial = "" + +[[state]] +name = "EmbeddingModel" +type = "string" +initial = "" + +[[vector]] +name = "embed" +property = "Embedding" +model_property = "EmbeddingModel" +dims = 2 +metric = "cosine" + [[action]] name = "Create" kind = "input" from = ["Draft"] to = "Active" -params = ["Name"] +params = ["Name", "Embedding", "EmbeddingModel"] [[action]] name = "Delete" @@ -1411,6 +1432,112 @@ async fn composite_atomic_batch_allows_existing_sub_write_to_delete_target() { ); } +#[cfg(feature = "sim")] +#[tokio::test] +async fn composite_dispatch_co_commits_vector_purge_fence_before_delayed_repair() { + let store = SimEventStore::no_faults(46); + let state = composite_test_state_with_store(store.clone()); + let tenant = TenantId::default(); + let agent = AgentContext::for_service("composite-vector-test"); + let child_id = "child-vector-through-composite"; + let child_fingerprint = state + .transition_tables + .get("Child") + .and_then(|table| table.spec_declaration_fingerprint.as_deref()) + .expect("Child table must carry its exact declaration fingerprint"); + let generation = store + .begin_vector_index_reconciliation("default", "Child", "v2|embed", 1, child_fingerprint) + .await + .expect("begin vector reconciliation"); + let stale_row = EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }; + + let created = state + .apply_composite_integration_result( + &tenant, + "Parent", + "parent-create-vector-child", + "CreateChild", + &json!({ + "sub_writes": [{ + "entity_type": "Child", + "entity_id": child_id, + "action": "Create", + "params": { + "Name": "vectored child", + "Embedding": "[1.0,0.0]", + "EmbeddingModel": "m1" + } + }] + }), + &agent, + ) + .await + .expect("composite create should co-commit the vector row"); + assert!(created); + assert_eq!( + store + .vector_candidates("default", "Child", "embed", "m1", 10) + .await + .expect("read composite-created vector"), + vec![temper_runtime::persistence::EntityVectorCandidate { + entity_id: child_id.to_string(), + vector: stale_row.vector.clone(), + }] + ); + + let deleted = state + .apply_composite_integration_result( + &tenant, + "Parent", + "parent-delete-vector-child", + "DeleteChild", + &json!({ + "sub_writes": [{ + "entity_type": "Child", + "entity_id": child_id, + "action": "Delete", + "params": {} + }] + }), + &agent, + ) + .await + .expect("composite delete should co-commit an empty vector set and fence"); + assert!(deleted); + + // The absent target bootstrap + Create are sequences 1 and 2; Delete is 3. + // Resume a repair that observed the pre-delete state only after Delete commits. + store + .backfill_entity_vectors( + "default", + "Child", + child_id, + generation, + 2, + std::slice::from_ref(&stale_row), + ) + .await + .expect("the delayed lower-sequence repair is a successful no-op"); + assert!( + store + .vector_candidates("default", "Child", "embed", "m1", 10) + .await + .expect("read post-delete vector partition") + .is_empty(), + "the real composite dispatch path must retain the sequence-3 purge fence" + ); + assert_eq!( + store + .dump_journal(&format!("default:Child:{child_id}")) + .len(), + 3 + ); +} + #[cfg(feature = "sim")] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn composite_ingest_pack_large_blob_sub_write_persists_overflow_fields() { diff --git a/crates/temper-server/src/state/entity_ops.rs b/crates/temper-server/src/state/entity_ops.rs index 633df47b1..b2be24636 100644 --- a/crates/temper-server/src/state/entity_ops.rs +++ b/crates/temper-server/src/state/entity_ops.rs @@ -205,6 +205,44 @@ impl ServerState { .map_err(|e| format!("registry lock poisoned: {e}")) } + /// Resolve a transition-table snapshot with tenant-aware legacy fallback. + /// + /// Once a tenant exists in the registry, that tenant config is authoritative: + /// an omitted type must not reappear through the boot-time compatibility map. + pub(crate) fn transition_table_for_tenant( + &self, + tenant: &TenantId, + entity_type: &str, + ) -> Result>, String> { + let registry = self + .registry + .read() + .map_err(|error| format!("registry lock poisoned: {error}"))?; + if registry.get_tenant(tenant).is_some() { + return Ok(registry.get_table(tenant, entity_type)); + } + Ok(self.transition_tables.get(entity_type).cloned()) + } + + /// Resolve the live transition-table lock with tenant-aware legacy fallback. + pub(crate) fn transition_table_live_for_tenant( + &self, + tenant: &TenantId, + entity_type: &str, + ) -> Result>>, String> { + let registry = self + .registry + .read() + .map_err(|error| format!("registry lock poisoned: {error}"))?; + if registry.get_tenant(tenant).is_some() { + return Ok(registry.get_table_live(tenant, entity_type)); + } + Ok(self + .transition_tables + .get(entity_type) + .map(|table| Arc::new(RwLock::new((**table).clone())))) + } + /// Returns `true` when dispatch should be allowed for the entity type. /// /// This includes both tenant-scoped specs and legacy single-tenant @@ -214,8 +252,9 @@ impl ServerState { tenant: &TenantId, entity_type: &str, ) -> Result { - Ok(self.has_registered_spec(tenant, entity_type)? - || self.transition_tables.contains_key(entity_type)) + Ok(self + .transition_table_for_tenant(tenant, entity_type)? + .is_some()) } /// Declared `[[key]]` set for a `(tenant, entity_type)` (ADR-0153), resolved @@ -236,14 +275,8 @@ impl ServerState { // Fail fast on a poisoned registry lock rather than silently falling through // to `transition_tables` — a silent fallback would re-introduce exactly the // ARN-68 bug (registry-installed keys not found → keyed path disabled → scan). - { - let registry = self.registry.read().expect("registry lock poisoned"); - if let Some(table) = registry.get_table(tenant, entity_type) { - return table.keys.clone(); - } - } - self.transition_tables - .get(entity_type) + self.transition_table_for_tenant(tenant, entity_type) + .expect("registry lock poisoned") .map(|table| table.keys.clone()) .unwrap_or_default() } @@ -258,14 +291,8 @@ impl ServerState { tenant: &TenantId, entity_type: &str, ) -> Vec { - { - let registry = self.registry.read().expect("registry lock poisoned"); - if let Some(table) = registry.get_table(tenant, entity_type) { - return table.vectors.clone(); - } - } - self.transition_tables - .get(entity_type) + self.transition_table_for_tenant(tenant, entity_type) + .expect("registry lock poisoned") .map(|table| table.vectors.clone()) .unwrap_or_default() } @@ -481,9 +508,10 @@ impl ServerState { projection_backfill::populate_key_index_from_snapshots(self, tenant).await; } - /// ADR-0155: backfill `entity_vector_index` for pre-existing entities of every - /// vector-declaring type and record the watermark. Idempotent; entities written - /// after boot maintain their vectors inline (co-commit) or write-behind. + /// ADR-0155/ADR-0181: reconcile `entity_vector_index` for pre-existing entities + /// of every current or previously covered vector-declaring type and record the + /// watermark. Idempotent; entities written after boot co-commit their journal, + /// retained vector sequence fence, and candidate rows. #[instrument(skip_all, fields(otel.name = "entity.populate_vector_index", tenant = %tenant))] pub async fn populate_vector_index_from_snapshots(&self, tenant: &TenantId) { projection_backfill::populate_vector_index_from_snapshots(self, tenant).await; @@ -691,6 +719,16 @@ impl ServerState { initial_fields: serde_json::Value, ) -> Option> { let key = format!("{tenant}:{entity_type}:{entity_id}"); + let _publication_guard = self + .actor_spec_publication_lock + .read() + .expect("actor spec publication lock poisoned"); + + // Resolve governance before consulting the actor cache. A removed type + // must not keep an orphan actor reachable through the fast path. + let table = self + .transition_table_live_for_tenant(tenant, entity_type) + .ok()??; // Fast-path: check actor registry under read lock. { @@ -701,21 +739,6 @@ impl ServerState { } } - // Look up live transition table reference: try SpecRegistry first, - // fall back to legacy map (wrapped in a fresh RwLock for compat). - let table = { - let reg = self.registry.read().unwrap(); - reg.get_table_live(tenant, entity_type) - } - .or_else(|| { - // Legacy single-tenant: wrap the static Arc in a - // new RwLock. Hot-swap doesn't apply to legacy mode, but the actor - // API is uniform. One clone per entity spawn (cheap). - self.transition_tables - .get(entity_type) - .map(|t| Arc::new(RwLock::new((**t).clone()))) - })?; - // Build actor instance (spawn guarded below to avoid duplicate races). // ADR-0048 sub-decision 5: every actor gets the shared idempotency // cache so it can dedupe duplicate asks produced by retry storms. @@ -798,6 +821,99 @@ impl ServerState { runtime_metrics::record_server_state_metrics(self); } + /// Snapshot ready actor incarnations for the specified tenant/type set. + #[cfg(feature = "observe")] + pub(crate) fn ready_actor_identities_for_types( + &self, + tenant: &TenantId, + entity_types: &[String], + ) -> BTreeMap { + let prefixes = entity_types + .iter() + .map(|entity_type| format!("{tenant}:{entity_type}:")) + .collect::>(); + self.actor_registry + .read() + .expect("actor registry lock poisoned during spec replacement") + .iter() + .filter_map(|(key, actor)| { + if !prefixes.iter().any(|prefix| key.starts_with(prefix)) { + return None; + } + actor + .ready_incarnation() + .map(|incarnation| (key.clone(), (actor.id().uid, incarnation))) + }) + .collect() + } + + /// Stop matching actors except keys known to predate durable publication. + #[cfg(feature = "observe")] + pub(crate) fn evict_type_actors_except( + &self, + tenant: &TenantId, + entity_types: &[String], + preserved_actors: &BTreeMap, + ) { + if entity_types.is_empty() { + return; + } + let prefixes = entity_types + .iter() + .map(|entity_type| format!("{tenant}:{entity_type}:")) + .collect::>(); + let removed = { + let mut actors = self + .actor_registry + .write() + .expect("actor registry lock poisoned during spec replacement"); + let keys = actors + .iter() + .filter(|(key, actor)| { + let preserved = preserved_actors.get(key.as_str()).is_some_and(|identity| { + actor + .ready_incarnation() + .is_some_and(|incarnation| *identity == (actor.id().uid, incarnation)) + }); + !preserved && prefixes.iter().any(|prefix| key.starts_with(prefix)) + }) + .map(|(key, _)| key.clone()) + .collect::>(); + keys.into_iter() + .filter_map(|key| actors.remove(&key).map(|actor| (key, actor))) + .collect::>() + }; + let mut last_accessed = self + .last_accessed + .write() + .expect("actor access registry lock poisoned during spec replacement"); + for (key, _) in &removed { + last_accessed.remove(key); + } + drop(last_accessed); + for (key, actor) in removed { + if let Err(error) = actor.stop() { + tracing::warn!( + tenant = %tenant, + actor_key = %key, + error = ?error, + "removed-type actor failed to stop after eviction" + ); + } + } + } + + /// Revalidate preserved actor identities after registry publication. + #[cfg(feature = "observe")] + pub(crate) fn revalidate_type_actors_after_publication( + &self, + tenant: &TenantId, + entity_types: &[String], + preserved_actors: &BTreeMap, + ) { + self.evict_type_actors_except(tenant, entity_types, preserved_actors); + } + /// Stop and evict an entity actor plus its in-memory indexes. /// /// Used after an out-of-band durable append (for example, an atomic @@ -1075,15 +1191,7 @@ impl ServerState { return Ok(None); } - let table = { - let reg = self.registry.read().unwrap(); - reg.get_table_live(tenant, entity_type) - } - .or_else(|| { - self.transition_tables - .get(entity_type) - .map(|t| Arc::new(RwLock::new((**t).clone()))) - }); + let table = self.transition_table_live_for_tenant(tenant, entity_type)?; let Some(table_ref) = table else { return Ok(None); }; @@ -1091,7 +1199,10 @@ impl ServerState { .read() .expect("transition table lock poisoned") .clone(); - if !table.rules.is_empty() { + // Vector rows and their declaration fence must be co-committed with the + // event journal. The native data-only shortcut does not expose that + // contract, so vector-declaring types use the normal EntityActor path. + if !table.rules.is_empty() || !table.vectors.is_empty() { return Ok(None); } @@ -1152,6 +1263,19 @@ impl ServerState { }; let projection_fields = self.query_projection_fields(tenant, entity_type, &state.fields); + let mut key_rows = Vec::new(); + if let Some(field_map) = state.fields.as_object() { + for key in &table.keys { + if let Some(key_hash) = + crate::key_index::canonical_key_hash(&key.name, &key.properties, field_map) + { + key_rows.push(temper_runtime::persistence::EntityKeyRow { + key_name: key.name.clone(), + key_hash, + }); + } + } + } let mut created_projection_state = state.clone(); created_projection_state.sequence_nr = 1; created_projection_state.push_event_bounded(created.clone()); @@ -1177,6 +1301,7 @@ impl ServerState { fields: &projection_fields, state: &projection_state, event: &envelope, + spec_declaration_fingerprint: table.spec_declaration_fingerprint.as_deref(), }) .instrument(native_span) .await @@ -1225,7 +1350,17 @@ impl ServerState { } else { let append_started_at = Instant::now(); // determinism-ok: production-only append wait metric let append_result = store - .append(&persistence_id, state.sequence_nr, &[envelope]) + .append_with_index_rows( + &persistence_id, + state.sequence_nr, + &[envelope], + crate::storage::AppendIndexRows { + key_rows: &key_rows, + vector_rows: &[], + reconcile_vectors: false, + spec_declaration_fingerprint: table.spec_declaration_fingerprint.as_deref(), + }, + ) .await; runtime_metrics::record_event_store_append_wait( backend.as_str(), diff --git a/crates/temper-server/src/state/file_initial_writes.rs b/crates/temper-server/src/state/file_initial_writes.rs index 90ba4df9c..779a2dd7a 100644 --- a/crates/temper-server/src/state/file_initial_writes.rs +++ b/crates/temper-server/src/state/file_initial_writes.rs @@ -1,5 +1,3 @@ -use std::sync::{Arc, RwLock}; - use temper_runtime::persistence::{EventMetadata, PersistenceEnvelope, PersistenceError}; use temper_runtime::scheduler::{sim_now, sim_uuid}; @@ -148,7 +146,38 @@ impl ServerState { .map(|(idx, event)| synthetic_envelope(&persistence_id, (idx + 1) as u64, event)) .collect::, _>>()?; - match store.append(&persistence_id, 0, &envelopes).await { + let mut key_rows = Vec::new(); + if let Some(field_map) = state.fields.as_object() { + for key in &table.keys { + if let Some(key_hash) = + crate::key_index::canonical_key_hash(&key.name, &key.properties, field_map) + { + key_rows.push(temper_runtime::persistence::EntityKeyRow { + key_name: key.name.clone(), + key_hash, + }); + } + } + } + let vector_rows = crate::vector_index::rows_for_entity_state( + &table.vectors, + &state.status, + &state.fields, + ); + match store + .append_with_index_rows( + &persistence_id, + 0, + &envelopes, + crate::storage::AppendIndexRows { + key_rows: &key_rows, + vector_rows: &vector_rows, + reconcile_vectors: !table.vectors.is_empty(), + spec_declaration_fingerprint: table.spec_declaration_fingerprint.as_deref(), + }, + ) + .await + { Ok(sequence_nr) => state.sequence_nr = sequence_nr, Err(PersistenceError::ConcurrencyViolation { .. }) => { return Err(FileStreamContentError::ActionRejected(format!( @@ -211,20 +240,14 @@ impl ServerState { &self, tenant: &temper_runtime::tenant::TenantId, ) -> Result { - let table = { - let reg = self.registry.read().unwrap(); - reg.get_table_live(tenant, "File") - } - .or_else(|| { - self.transition_tables - .get("File") - .map(|t| Arc::new(RwLock::new((**t).clone()))) - }) - .ok_or_else(|| { - FileStreamContentError::State(format!( - "No transition table for tenant '{tenant}', entity type 'File'" - )) - })?; + let table = self + .transition_table_live_for_tenant(tenant, "File") + .map_err(FileStreamContentError::State)? + .ok_or_else(|| { + FileStreamContentError::State(format!( + "No transition table for tenant '{tenant}', entity type 'File'" + )) + })?; Ok(table .read() diff --git a/crates/temper-server/src/state/mod.rs b/crates/temper-server/src/state/mod.rs index e9cd9796d..2284ab879 100644 --- a/crates/temper-server/src/state/mod.rs +++ b/crates/temper-server/src/state/mod.rs @@ -487,6 +487,20 @@ pub struct ServerState { /// between "check" and "write" while cross-actor transactions are still /// being built out. pub(crate) commons_write_guardrail_lock: Arc>, + /// Serializes vector declaration snapshotting and durable reconciliation- + /// generation allocation. The durable declaration revision and store generation + /// remain authoritative across crashes/processes; this short critical section + /// prevents an older local snapshot from beginning after a hot swap (ADR-0181). + pub(crate) vector_reconciliation_lock: Arc>, + /// Serializes durable spec-catalog mutation with registry publication. + /// + /// Without this lock, concurrent full replacements can each compute omissions + /// from the same old registry and leave storage and memory with different truth. + #[cfg(feature = "observe")] + pub(crate) spec_catalog_update_lock: Arc>, + /// Serializes actor table capture/insertion with spec publication and + /// removed-type actor eviction. + pub(crate) actor_spec_publication_lock: Arc>, pub secrets_vault: Option>, /// Broadcast channel for agent progress events (SSE subscriptions). /// // determinism-ok: broadcast channel for external observation only @@ -714,6 +728,10 @@ impl ServerState { commons_rate_limit_buckets: Arc::new(Mutex::new(BTreeMap::new())), commons_storage_projection_cache: Arc::new(Mutex::new(BTreeMap::new())), commons_write_guardrail_lock: Arc::new(tokio::sync::Mutex::new(())), + vector_reconciliation_lock: Arc::new(tokio::sync::Mutex::new(())), + #[cfg(feature = "observe")] + spec_catalog_update_lock: Arc::new(tokio::sync::Mutex::new(())), + actor_spec_publication_lock: Arc::new(RwLock::new(())), secrets_vault: None, agent_progress_tx: Arc::new(agent_progress_tx), // determinism-ok: broadcast for external observation entity_event_sequences: Arc::new(Mutex::new(BTreeMap::new())), @@ -845,14 +863,28 @@ impl ServerState { csdl_xml: String, ioa_sources: BTreeMap, ) -> Result { - let mut state = Self::new(system, csdl, csdl_xml); - let mut tables = BTreeMap::new(); - for (entity_type, ioa_source) in &ioa_sources { - let table = TransitionTable::try_from_ioa_source(ioa_source) - .map_err(|e| format!("entity '{entity_type}': {e}"))?; - tables.insert(entity_type.clone(), Arc::new(table)); + let tenant = TenantId::default(); + let ioa_refs = ioa_sources + .iter() + .map(|(entity_type, ioa_source)| (entity_type.as_str(), ioa_source.as_str())) + .collect::>(); + let mut registry = SpecRegistry::new(); + registry + .try_register_tenant(tenant.clone(), csdl.clone(), csdl_xml.clone(), &ioa_refs) + .map_err(|error| error.to_string())?; + for entity_type in ioa_sources.keys() { + registry.remove_verification_status(&tenant, entity_type); } + let tables = registry + .get_tenant(&tenant) + .ok_or_else(|| "default tenant registration did not produce a config".to_string())? + .entities + .iter() + .map(|(entity_type, spec)| (entity_type.clone(), spec.table())) + .collect(); + let mut state = Self::new(system, csdl, csdl_xml); state.transition_tables = Arc::new(tables); + state.registry = Arc::new(RwLock::new(registry)); Ok(state) } @@ -962,6 +994,10 @@ impl ServerState { commons_rate_limit_buckets: Arc::new(Mutex::new(BTreeMap::new())), commons_storage_projection_cache: Arc::new(Mutex::new(BTreeMap::new())), commons_write_guardrail_lock: Arc::new(tokio::sync::Mutex::new(())), + vector_reconciliation_lock: Arc::new(tokio::sync::Mutex::new(())), + #[cfg(feature = "observe")] + spec_catalog_update_lock: Arc::new(tokio::sync::Mutex::new(())), + actor_spec_publication_lock: Arc::new(RwLock::new(())), secrets_vault: None, agent_progress_tx: Arc::new(agent_progress_tx), // determinism-ok: broadcast for external observation entity_event_sequences: Arc::new(Mutex::new(BTreeMap::new())), @@ -1145,6 +1181,28 @@ impl ServerState { provider.store_for_tenant(tenant).await } + /// Return the platform persistence capability that owns one tenant's specs. + /// + /// Tenant-routed Turso uses a distinct store per tenant. Shared backends such + /// as Postgres expose one [`crate::platform_store::PlatformStore`] whose rows + /// are tenant-scoped. Bootstrap and verification code must use this helper + /// instead of assuming that durable platform metadata is always Turso. + pub async fn platform_store_for_tenant( + &self, + tenant: &str, + ) -> Option> { + if let Some(turso) = self.turso_store_for_tenant(tenant).await { + return Some(Arc::new(turso)); + } + let stack = self.storage_stack.as_ref()?; + if stack.backend == BackendLabel::TursoRouted + && !matches!(tenant, "temper-system" | "default") + { + return None; + } + stack.platform.clone() + } + /// Return a backend-neutral metadata store for one tenant. /// /// Postgres is a shared platform store with tenant columns; Turso may be @@ -1435,7 +1493,11 @@ impl ServerState { #[cfg(test)] mod tests { - use super::normalize_local_tdata_host; + use temper_runtime::ActorSystem; + use temper_store_turso::TenantStoreRouter; + + use super::{ServerState, normalize_local_tdata_host}; + use crate::{SpecRegistry, StorageStack}; #[test] fn normalize_local_tdata_host_accepts_urls_domains_and_ports() { @@ -1459,4 +1521,37 @@ mod tests { assert_eq!(normalize_local_tdata_host("https:///tdata"), None); assert_eq!(normalize_local_tdata_host("bad host.example"), None); } + + #[tokio::test] + async fn routed_platform_store_does_not_fallback_for_unknown_tenant() { + let router = TenantStoreRouter::new( + "file:/tmp/temper-arn216-routed-platform-store.db", + None, + None, + ) + .await + .expect("create routed Turso store"); + let mut state = + ServerState::from_registry(ActorSystem::new("routed-store"), SpecRegistry::new()); + state.set_storage_stack(StorageStack::from_tenant_router(router)); + + assert!( + state + .platform_store_for_tenant("unregistered-tenant") + .await + .is_none(), + "an unknown routed tenant must not write specs into the shared platform database" + ); + assert!( + state + .platform_store_for_tenant("temper-system") + .await + .is_some(), + "the system tenant is explicitly owned by the platform database" + ); + assert!( + state.platform_store_for_tenant("default").await.is_some(), + "the reserved default tenant remains explicitly platform-backed" + ); + } } diff --git a/crates/temper-server/src/state/persistence/mod.rs b/crates/temper-server/src/state/persistence/mod.rs index b41acf070..0b07d90a3 100644 --- a/crates/temper-server/src/state/persistence/mod.rs +++ b/crates/temper-server/src/state/persistence/mod.rs @@ -19,6 +19,7 @@ pub(crate) enum TenantMetadataBackend { } mod logs_and_secrets; +mod spec_catalog; mod spec_metadata; const BUNDLED_REPLACE_UPLOAD_SOURCE: &str = "bundled-replace-upload"; diff --git a/crates/temper-server/src/state/persistence/spec_catalog.rs b/crates/temper-server/src/state/persistence/spec_catalog.rs new file mode 100644 index 000000000..fd7eeac9f --- /dev/null +++ b/crates/temper-server/src/state/persistence/spec_catalog.rs @@ -0,0 +1,118 @@ +#[cfg(feature = "observe")] +use std::collections::{BTreeMap, BTreeSet}; + +use super::ServerState; +#[cfg(feature = "observe")] +use super::TenantMetadataBackend; + +impl ServerState { + /// Atomically promote the exact verified staged catalog and its omissions. + #[cfg(feature = "observe")] + pub(crate) async fn persist_verified_spec_catalog_update( + &self, + tenant: &str, + ioa_sources: &BTreeMap, + csdl_xml: &str, + additional_removed_entity_types: &[String], + replace: bool, + cross_invariants_toml: Option<&str>, + ) -> Result, String> { + let fingerprints = ioa_sources + .iter() + .map(|(entity_type, source)| { + ( + entity_type.as_str(), + temper_store_turso::spec_content_hash(source), + csdl_xml, + ) + }) + .collect::>(); + let expected = fingerprints + .iter() + .map(|(entity_type, fingerprint, csdl)| (*entity_type, fingerprint.as_str(), *csdl)) + .collect::>(); + let incoming = ioa_sources + .keys() + .map(String::as_str) + .collect::>(); + let mut removed_entity_types = additional_removed_entity_types + .iter() + .filter(|entity_type| !incoming.contains(entity_type.as_str())) + .cloned() + .collect::>(); + + match self.tenant_metadata_backend(tenant).await { + Some(TenantMetadataBackend::Postgres(pool)) => { + removed_entity_types.extend( + temper_store_postgres::PostgresEventStore::new(pool) + .persist_verified_spec_catalog_update( + tenant, + &expected, + additional_removed_entity_types, + replace, + cross_invariants_toml, + ) + .await + .map_err(|error| error.to_string())?, + ); + } + Some(TenantMetadataBackend::Turso(store)) => { + removed_entity_types.extend( + store + .persist_verified_spec_catalog_update( + tenant, + &expected, + additional_removed_entity_types, + replace, + cross_invariants_toml, + ) + .await + .map_err(|error| error.to_string())?, + ); + } + Some(TenantMetadataBackend::Redis) => { + return Err(Self::redis_ephemeral_error( + "Verified spec catalog publication", + )); + } + None if replace => { + if let Some((store, _)) = self.event_journal() { + removed_entity_types.extend( + store + .spec_declaration_entity_types(tenant) + .await + .map_err(|error| error.to_string())? + .into_iter() + .filter(|entity_type| !incoming.contains(entity_type.as_str())), + ); + } + } + None => {} + } + + for (entity_type, fingerprint, _) in &fingerprints { + self.persist_event_store_spec_declaration(tenant, entity_type, fingerprint) + .await?; + } + for entity_type in &removed_entity_types { + self.persist_event_store_spec_declaration(tenant, entity_type, "absent:v1") + .await?; + } + Ok(removed_entity_types.into_iter().collect()) + } + + pub(super) async fn persist_event_store_spec_declaration( + &self, + tenant: &str, + entity_type: &str, + declaration_fingerprint: &str, + ) -> Result<(), String> { + if let Some((store, _)) = self.event_journal() { + store + .persist_spec_declaration(tenant, entity_type, declaration_fingerprint) + .await + .map_err(|error| error.to_string())?; + } + Ok(()) + } +} diff --git a/crates/temper-server/src/state/persistence/spec_metadata.rs b/crates/temper-server/src/state/persistence/spec_metadata.rs index 3f544d816..c5be08f8a 100644 --- a/crates/temper-server/src/state/persistence/spec_metadata.rs +++ b/crates/temper-server/src/state/persistence/spec_metadata.rs @@ -5,54 +5,165 @@ use super::super::ServerState; use super::TenantMetadataBackend; use crate::registry::EntityVerificationResult; +async fn stage_postgres_spec_source( + pool: &sqlx::PgPool, + tenant: &str, + entity_type: &str, + ioa_source: &str, + csdl_xml: &str, + content_hash: &str, +) -> Result<(), sqlx::Error> { + sqlx::query( + "INSERT INTO staged_specs \ + (tenant, entity_type, ioa_source, csdl_xml, content_hash, version, updated_at) \ + VALUES ($1, $2, $3, $4, $5, 1, now()) \ + ON CONFLICT (tenant, entity_type) DO UPDATE SET \ + ioa_source = EXCLUDED.ioa_source, \ + csdl_xml = EXCLUDED.csdl_xml, \ + content_hash = EXCLUDED.content_hash, \ + version = CASE \ + WHEN staged_specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash \ + OR staged_specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml \ + THEN staged_specs.version + 1 \ + ELSE staged_specs.version \ + END, \ + updated_at = CASE \ + WHEN staged_specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash \ + OR staged_specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml \ + THEN now() \ + ELSE staged_specs.updated_at \ + END", + ) + .bind(tenant) + .bind(entity_type) + .bind(ioa_source) + .bind(csdl_xml) + .bind(content_hash) + .execute(pool) + .await + .map(|_| ()) +} + +async fn delete_postgres_spec_source( + pool: &sqlx::PgPool, + tenant: &str, + entity_type: &str, +) -> Result<(), sqlx::Error> { + sqlx::query( + "WITH staged AS ( \ + DELETE FROM staged_specs WHERE tenant = $1 AND entity_type = $2 \ + ) SELECT tombstone_spec_declaration_authority($1, $2)", + ) + .bind(tenant) + .bind(entity_type) + .execute(pool) + .await + .map(|_| ()) +} + impl ServerState { - /// Upsert a spec source into the persistence backend (Postgres or Turso). - pub async fn upsert_spec_source( + /// Stage candidate catalog bytes without changing committed authority. + #[cfg(feature = "observe")] + pub(crate) async fn stage_spec_catalog_update( &self, tenant: &str, - entity_type: &str, - ioa_source: &str, + ioa_sources: &std::collections::BTreeMap, csdl_xml: &str, ) -> Result<(), String> { let Some(backend) = self.tenant_metadata_backend(tenant).await else { return Ok(()); }; - - match backend { - TenantMetadataBackend::Postgres(pool) => { - sqlx::query( - "INSERT INTO specs \ - (tenant, entity_type, ioa_source, csdl_xml, version, verified, verification_status, updated_at) \ - VALUES ($1, $2, $3, $4, 1, false, 'pending', now()) \ - ON CONFLICT (tenant, entity_type) DO UPDATE SET \ - ioa_source = EXCLUDED.ioa_source, \ - csdl_xml = EXCLUDED.csdl_xml, \ - version = specs.version + 1, \ - verified = false, \ - verification_status = 'pending', \ - levels_passed = NULL, \ - levels_total = NULL, \ - verification_result = NULL, \ - updated_at = now()", + for (entity_type, ioa_source) in ioa_sources { + let content_hash = temper_store_turso::spec_content_hash(ioa_source); + match &backend { + TenantMetadataBackend::Postgres(pool) => stage_postgres_spec_source( + pool, + tenant, + entity_type, + ioa_source, + csdl_xml, + &content_hash, ) - .bind(tenant) - .bind(entity_type) - .bind(ioa_source) - .bind(csdl_xml) - .execute(&pool) .await - .map(|_| ()) - .map_err(|e| format!("failed to upsert spec {tenant}/{entity_type} in postgres: {e}")) - } - TenantMetadataBackend::Turso(turso) => { - let hash = temper_store_turso::spec_content_hash(ioa_source); - turso - .upsert_spec(tenant, entity_type, ioa_source, csdl_xml, &hash) + .map_err(|error| { + format!("failed to stage spec {tenant}/{entity_type} in postgres: {error}") + })?, + TenantMetadataBackend::Turso(store) => store + .upsert_spec(tenant, entity_type, ioa_source, csdl_xml, &content_hash) .await - .map_err(|e| format!("failed to upsert spec {tenant}/{entity_type} in turso: {e}")) + .map_err(|error| { + format!("failed to stage spec {tenant}/{entity_type} in turso: {error}") + })?, + TenantMetadataBackend::Redis => { + return Err(Self::redis_ephemeral_error("Spec source staging")); + } } - TenantMetadataBackend::Redis => Err(Self::redis_ephemeral_error("Spec source persistence")), } + Ok(()) + } + + /// Upsert a spec source into the persistence backend (Postgres or Turso). + pub async fn upsert_spec_source( + &self, + tenant: &str, + entity_type: &str, + ioa_source: &str, + csdl_xml: &str, + ) -> Result<(), String> { + let content_hash = temper_store_turso::spec_content_hash(ioa_source); + if let Some(backend) = self.tenant_metadata_backend(tenant).await { + match backend { + TenantMetadataBackend::Postgres(pool) => stage_postgres_spec_source( + &pool, + tenant, + entity_type, + ioa_source, + csdl_xml, + &content_hash, + ) + .await + .map_err(|e| { + format!("failed to upsert spec {tenant}/{entity_type} in postgres: {e}") + }), + TenantMetadataBackend::Turso(turso) => turso + .upsert_spec(tenant, entity_type, ioa_source, csdl_xml, &content_hash) + .await + .map_err(|e| { + format!("failed to upsert spec {tenant}/{entity_type} in turso: {e}") + }), + TenantMetadataBackend::Redis => { + Err(Self::redis_ephemeral_error("Spec source persistence")) + } + }?; + } + self.persist_event_store_spec_declaration(tenant, entity_type, &content_hash) + .await + } + + /// Delete a persisted spec source while retaining the backend's declaration + /// tombstone used to fence stale writers and resume vector-row purging. + pub async fn delete_spec_source(&self, tenant: &str, entity_type: &str) -> Result<(), String> { + if let Some(backend) = self.tenant_metadata_backend(tenant).await { + match backend { + TenantMetadataBackend::Postgres(pool) => { + delete_postgres_spec_source(&pool, tenant, entity_type) + .await + .map_err(|e| { + format!("failed to delete spec {tenant}/{entity_type} in postgres: {e}") + }) + } + TenantMetadataBackend::Turso(turso) => { + turso.delete_spec(tenant, entity_type).await.map_err(|e| { + format!("failed to delete spec {tenant}/{entity_type} in turso: {e}") + }) + } + TenantMetadataBackend::Redis => { + Err(Self::redis_ephemeral_error("Spec source deletion")) + } + }?; + } + self.persist_event_store_spec_declaration(tenant, entity_type, "absent:v1") + .await } /// Upsert tenant-level cross-invariant definitions. @@ -192,3 +303,79 @@ impl ServerState { } } } + +#[cfg(test)] +mod tests { + use temper_store_postgres::{ + PostgresEventStore, PostgresSpecVerificationUpdate, migration::run_migrations, + }; + + use super::{delete_postgres_spec_source, stage_postgres_spec_source}; + + #[test] + fn postgres_hot_update_and_delete_fence_stale_verification() { + let Ok(database_url) = std::env::var("DATABASE_URL") else { + return; + }; + sqlx::__rt::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("tenant-server-spec-race-{}", uuid::Uuid::new_v4()); + let ioa_a = "[automaton]\nname = \"Item\"\n# a\n"; + let ioa_b = "[automaton]\nname = \"Item\"\n# b\n"; + let csdl = ""; + let hash_a = temper_store_turso::spec_content_hash(ioa_a); + let hash_b = temper_store_turso::spec_content_hash(ioa_b); + let verified = || PostgresSpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }; + + stage_postgres_spec_source(&pool, &tenant, "Item", ioa_a, csdl, &hash_a) + .await + .expect("stage A through server path"); + store + .commit_verified_spec(&tenant, "Item", &hash_a, csdl, verified()) + .await + .expect("commit A"); + stage_postgres_spec_source(&pool, &tenant, "Item", ioa_b, csdl, &hash_b) + .await + .expect("stage B through server path"); + store + .commit_verified_spec(&tenant, "Item", &hash_a, csdl, verified()) + .await + .expect_err("stale A verification must not publish B"); + + let committed: (String,) = sqlx::query_as( + "SELECT content_hash FROM specs WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(&pool) + .await + .expect("read committed A"); + assert_eq!(committed.0, hash_a); + + delete_postgres_spec_source(&pool, &tenant, "Item") + .await + .expect("delete through server path"); + store + .commit_verified_spec(&tenant, "Item", &hash_b, csdl, verified()) + .await + .expect_err("stale B verification must not resurrect deletion"); + let remaining: (i64,) = sqlx::query_as( + "SELECT \ + (SELECT COUNT(*) FROM specs WHERE tenant = $1 AND entity_type = 'Item') + \ + (SELECT COUNT(*) FROM staged_specs WHERE tenant = $1 AND entity_type = 'Item')", + ) + .bind(&tenant) + .fetch_one(&pool) + .await + .expect("count remaining catalog rows"); + assert_eq!(remaining.0, 0); + }); + } +} diff --git a/crates/temper-server/src/state/projection_backfill.rs b/crates/temper-server/src/state/projection_backfill.rs index 72f511c83..0666082f5 100644 --- a/crates/temper-server/src/state/projection_backfill.rs +++ b/crates/temper-server/src/state/projection_backfill.rs @@ -21,18 +21,11 @@ pub(super) fn transition_table_for( tenant: &TenantId, entity_type: &str, ) -> Option { - { - let registry = state.registry.read().unwrap(); - registry - .get_table_live(tenant, entity_type) - .map(|table| table.read().expect("table lock poisoned").clone()) - } - .or_else(|| { - state - .transition_tables - .get(entity_type) - .map(|table| (**table).clone()) - }) + state + .transition_table_for_tenant(tenant, entity_type) + .ok() + .flatten() + .map(|table| (*table).clone()) } /// Outcome of loading one entity's current state for an index backfill (ADR-0153, @@ -40,10 +33,14 @@ pub(super) fn transition_table_for( /// same way — the distinction is the watermark soundness gate. pub(super) enum EntityLoadOutcome { /// Loaded — index it from these fields. - Fields(serde_json::Value), + Fields { + fields: serde_json::Value, + status: String, + sequence_nr: u64, + }, /// Definitively skippable: deleted, or a phantom with no events. Correctly NOT /// indexed, and NOT a failure (it must not block the watermark). - Skip, + Skip { sequence_nr: u64 }, /// The entity exists (it was enumerated from the durable store) but its current /// state could not be loaded — no transition table to replay with, an unreadable /// snapshot, or a replay error. Indexing it is impossible, so the type must NOT be @@ -82,9 +79,17 @@ pub(super) async fn load_entity_current_fields( .await { Err(_) => EntityLoadOutcome::LoadFailed, - Ok(state) if state.status == "Deleted" => EntityLoadOutcome::Skip, - Ok(state) if state.total_event_count == 0 => EntityLoadOutcome::Skip, - Ok(state) => EntityLoadOutcome::Fields(state.fields), + Ok(state) if state.status == "Deleted" => EntityLoadOutcome::Skip { + sequence_nr: state.sequence_nr, + }, + Ok(state) if state.total_event_count == 0 => EntityLoadOutcome::Skip { + sequence_nr: state.sequence_nr, + }, + Ok(state) => EntityLoadOutcome::Fields { + fields: state.fields, + status: state.status, + sequence_nr: state.sequence_nr, + }, } } diff --git a/crates/temper-server/src/state/projection_backfill/key_index.rs b/crates/temper-server/src/state/projection_backfill/key_index.rs index 22b47a405..f8118dabe 100644 --- a/crates/temper-server/src/state/projection_backfill/key_index.rs +++ b/crates/temper-server/src/state/projection_backfill/key_index.rs @@ -141,7 +141,7 @@ pub(in crate::state) async fn populate_key_index_from_snapshots( ) .await { - EntityLoadOutcome::Fields(fields) => { + EntityLoadOutcome::Fields { fields, .. } => { let Some(field_map) = fields.as_object() else { skipped += 1; continue; @@ -179,7 +179,7 @@ pub(in crate::state) async fn populate_key_index_from_snapshots( } } } - EntityLoadOutcome::Skip => skipped += 1, + EntityLoadOutcome::Skip { .. } => skipped += 1, EntityLoadOutcome::LoadFailed => { failed += 1; tracing::warn!( diff --git a/crates/temper-server/src/state/projection_backfill/vector_index.rs b/crates/temper-server/src/state/projection_backfill/vector_index.rs index 44d0de6fe..f01e0c643 100644 --- a/crates/temper-server/src/state/projection_backfill/vector_index.rs +++ b/crates/temper-server/src/state/projection_backfill/vector_index.rs @@ -1,25 +1,55 @@ -//! ADR-0155 declared-vector backfill: populate `entity_vector_index` for entities -//! that existed before the `[[vector]]` path was declared (or, on a write-behind -//! backend, that lag the index), and record the per-(tenant, entity_type) watermark. +//! ADR-0181 sequence-monotonic vector-index reconciliation. //! -//! Mirrors the declared-key backfill (`key_index.rs`): authoritative enumeration -//! (registry types + `store.list_entity_ids_by_type`), strict state load, per-decl -//! vector parse, idempotent upsert, and a watermark only when every existing entity -//! was indexed or is definitively skippable. +//! Every repair enumerates durable journal streams (including deleted entities), +//! rebuilds current rows from a strict replay, and carries that replay's journal +//! sequence into a store-level compare-and-reconcile transaction. Completion is +//! watermarked only after every stream converges durably. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; +use temper_runtime::persistence::PersistenceError; use temper_runtime::tenant::TenantId; -use crate::ServerState; +use crate::{ServerState, storage::BoxedEventStore}; -use super::{EntityLoadOutcome, load_entity_current_fields, transition_table_for}; +use super::{EntityLoadOutcome, load_entity_current_fields}; + +fn vector_backfill_work_types( + current_types: &BTreeSet, + covered: &BTreeMap, + reconciliation_types: &BTreeSet, +) -> BTreeSet { + let mut work_types = current_types.clone(); + work_types.extend(covered.keys().cloned()); + work_types.extend(reconciliation_types.iter().cloned()); + work_types +} + +async fn durable_stream_sequence( + store: &BoxedEventStore, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, +) -> Result { + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let snapshot_sequence = store + .load_snapshot(&persistence_id) + .await? + .map(|(sequence_nr, _)| sequence_nr) + .unwrap_or(0); + let events = store + .read_events(&persistence_id, snapshot_sequence) + .await?; + Ok(events + .last() + .map(|event| event.sequence_nr) + .unwrap_or(snapshot_sequence)) +} /// Backfill `entity_vector_index` for existing entities, then record the watermark. /// -/// Idempotent; entities written after the vector path was declared already maintain -/// their vectors at write time (co-commit on postgres/sim, write-behind on turso). -/// Runs as a cooperative background task off the boot path. +/// Idempotent and safe alongside live writes: a rebuild observed at sequence N is +/// ignored by the store after a live append advances that entity's fence to N+1. pub(in crate::state) async fn populate_vector_index_from_snapshots( state: &ServerState, tenant: &TenantId, @@ -28,210 +58,371 @@ pub(in crate::state) async fn populate_vector_index_from_snapshots( return; }; - // Types with a declared vector path, from the registry (os-app entities live here). - let vectored_types: Vec<(String, Vec)> = { - let registry = state.registry.read().unwrap(); - registry - .entity_types(tenant) - .into_iter() - .filter_map(|entity_type| { - let table = registry.get_table(tenant, entity_type)?; - if table.vectors.is_empty() { - None - } else { - Some((entity_type.to_string(), table.vectors.clone())) - } - }) - .collect() + let covered: BTreeMap = match store + .vector_index_backfilled_types(tenant.as_str()) + .await + { + Ok(types) => types.into_iter().collect(), + Err(error) => { + tracing::error!( + tenant = %tenant, + error = %error, + "vector index backfill: failed to load durable watermarks; reconciliation aborted" + ); + return; + } }; - if vectored_types.is_empty() { - return; + let reconciliation_types: BTreeSet = match store + .vector_reconciliation_entity_types(tenant.as_str()) + .await + { + Ok(types) => types.into_iter().collect(), + Err(error) => { + tracing::error!( + tenant = %tenant, + error = %error, + "vector index backfill: failed to load durable reconciliation types; reconciliation aborted" + ); + return; + } + }; + + // The work set needs only type names. Declarations themselves are snapshotted + // later under the short snapshot+generation critical section. + let (mut current_types, uses_legacy_tables): (BTreeSet, bool) = { + let registry = state + .registry + .read() + .expect("spec registry lock poisoned while listing vector declarations"); + ( + registry + .entity_types(tenant) + .into_iter() + .map(ToString::to_string) + .collect(), + registry.get_tenant(tenant).is_none(), + ) + }; + if uses_legacy_tables { + current_types.extend(state.transition_tables.keys().cloned()); } - // The covered vector-path set per type (empty map on any failure — treat as - // never-backfilled, which is safe: it re-indexes, never skips wrongly). - let covered: std::collections::BTreeMap = store - .vector_index_backfilled_types(tenant.as_str()) - .await - .unwrap_or_default() - .into_iter() - .collect(); - - for (entity_type, vectors) in &vectored_types { - let current_set = crate::vector_index::declared_vector_set_signature(vectors); - // Already complete for the CURRENT declared vector-set: the write path keeps - // the index whole (co-commit) or write-behind + this backfill did, so skip. - if covered.get(entity_type).map(String::as_str) == Some(current_set.as_str()) { + let work_types = vector_backfill_work_types(¤t_types, &covered, &reconciliation_types); + + for entity_type in work_types { + // Serialize only declaration snapshot + durable generation allocation. + // Replaying journals and writing rows happens after this guard is released, + // so unrelated tenants and entity types are not blocked by a long rebuild. + let reconciliation_guard = state.vector_reconciliation_lock.lock().await; + let (table, declaration_revision, declaration_fingerprint) = { + let registry = state + .registry + .read() + .expect("spec registry lock poisoned during vector reconciliation"); + if let Some(config) = registry.get_tenant(tenant) { + if let Some(spec) = config.entities.get(&entity_type) { + let table = spec.table(); + let fingerprint = table + .spec_declaration_fingerprint + .clone() + .unwrap_or_else(|| temper_store_turso::spec_content_hash(&spec.ioa_source)); + (Some(table), config.revision, fingerprint) + } else { + (None, config.revision, "absent:v1".to_string()) + } + } else if let Some(table) = state.transition_tables.get(&entity_type).cloned() { + let fingerprint = table + .spec_declaration_fingerprint + .clone() + .unwrap_or_else(|| "absent:v1".to_string()); + (Some(table), 1, fingerprint) + } else { + (None, 1, "absent:v1".to_string()) + } + }; + let vectors = table + .as_deref() + .map(|table| table.vectors.clone()) + .unwrap_or_default(); + if vectors.is_empty() + && !covered.contains_key(&entity_type) + && !reconciliation_types.contains(&entity_type) + { continue; } - // A watermark covering a DIFFERENT set means a vector path was declared after - // the first backfill; re-index every existing entity under all current paths. - let force_full_reindex = covered.contains_key(entity_type); - if force_full_reindex { + let current_set = crate::vector_index::declared_vector_set_signature(&vectors); + if let Some(previous_set) = covered.get(&entity_type) + && previous_set != ¤t_set + { tracing::info!( - tenant = %tenant, entity_type = %entity_type, - covered_set = covered.get(entity_type).map(String::as_str).unwrap_or(""), + tenant = %tenant, + entity_type = %entity_type, + covered_set = %previous_set, current_set = %current_set, - "vector index backfill: declared vector-set changed — re-indexing every existing entity of this type (one-time)" + "vector index backfill: reconciliation signature changed; rebuilding every durable stream" ); } - let entity_ids = match store - .list_entity_ids_by_type(tenant.as_str(), entity_type) + let reconciliation_generation = match store + .begin_vector_index_reconciliation( + tenant.as_str(), + &entity_type, + ¤t_set, + declaration_revision, + &declaration_fingerprint, + ) .await { - Ok(ids) => ids, - Err(e) => { + Ok(generation) => generation, + Err(error) => { tracing::error!( - tenant = %tenant, entity_type = %entity_type, error = %e, - "vector index backfill: failed to enumerate entities; type not watermarked" + tenant = %tenant, + entity_type = %entity_type, + vector_set = %current_set, + error = %error, + "vector index backfill: failed to begin durable reconciliation generation" ); continue; } }; - // Resumability: on a first-time backfill, skip entities already indexed. On a - // set change, re-index all (a newly declared path is not yet on them). - let already_indexed: BTreeSet = if force_full_reindex { - BTreeSet::new() - } else { - match store - .vectored_entity_ids_for_type(tenant.as_str(), entity_type) - .await - { - Ok(ids) => ids.into_iter().collect(), - Err(_) => BTreeSet::new(), + // A cached watermark cannot be trusted before the declaration barrier: + // spec persistence may have withdrawn it after the initial tenant-wide + // read. Re-read after `begin` while coordinators are serialized. An exact + // retry keeps the watermark; a new declaration generation removes it. + let already_complete = match store.vector_index_backfilled_types(tenant.as_str()).await { + Ok(types) => types.into_iter().any(|(completed_type, completed_set)| { + completed_type == entity_type && completed_set == current_set + }), + Err(error) => { + tracing::error!( + tenant = %tenant, + entity_type = %entity_type, + error = %error, + "vector index backfill: failed to revalidate completion after declaration barrier" + ); + continue; + } + }; + drop(reconciliation_guard); + if already_complete { + continue; + } + + let entity_ids = match store + .list_vector_repair_entity_ids(tenant.as_str(), &entity_type) + .await + { + Ok(ids) => ids, + Err(error) => { + tracing::error!( + tenant = %tenant, + entity_type = %entity_type, + error = %error, + "vector index backfill: failed to enumerate durable streams; type not watermarked" + ); + continue; } }; - let table = transition_table_for(state, tenant, entity_type); let blob_store = state.blob_store_for_tenant(tenant).ok(); let total = entity_ids.len(); - let mut newly_indexed = 0usize; - let mut already = 0usize; - let mut skipped = 0usize; + let mut indexed = 0usize; + let mut empty = 0usize; let mut failed = 0usize; for entity_id in &entity_ids { - if already_indexed.contains(entity_id) { - already += 1; + if table.is_none() { + match durable_stream_sequence(&store, tenant, &entity_type, entity_id).await { + Ok(sequence_nr) => { + match store + .backfill_entity_vectors( + tenant.as_str(), + &entity_type, + entity_id, + reconciliation_generation, + sequence_nr, + &[], + ) + .await + { + Ok(()) => empty += 1, + Err(error) => { + failed += 1; + tracing::warn!( + error = %error, + entity_type = %entity_type, + entity_id = %entity_id, + sequence_nr, + "vector index backfill: absent-declaration purge failed" + ); + } + } + } + Err(error) => { + failed += 1; + tracing::warn!( + error = %error, + entity_type = %entity_type, + entity_id = %entity_id, + "vector index backfill: absent-declaration stream sequence could not be loaded" + ); + } + } + tokio::task::yield_now().await; continue; } match load_entity_current_fields( tenant, - entity_type, + &entity_type, entity_id, - table.as_ref(), + table.as_deref(), &store, backend, blob_store.as_ref(), ) .await { - EntityLoadOutcome::Fields(fields) => { - let Some(field_map) = fields.as_object() else { - skipped += 1; - continue; - }; - let mut vector_rows = Vec::new(); - for decl in vectors { - let Some(vector) = field_map - .get(&decl.property) - .and_then(|v| crate::vector_index::parse_vector_property(v, decl.dims)) - else { - continue; - }; - let Some(model_tag) = field_map - .get(&decl.model_property) - .and_then(|v| v.as_str()) - .filter(|tag| !tag.is_empty()) - else { - continue; - }; - vector_rows.push(temper_runtime::persistence::EntityVectorRow { - decl_name: decl.name.clone(), - model_tag: model_tag.to_string(), - vector, - }); - } - if vector_rows.is_empty() { - // No usable vector on this entity yet (unembedded) — not a - // failure; it is simply absent from the ranking until embedded. - skipped += 1; - continue; - } + EntityLoadOutcome::Fields { + fields, + status, + sequence_nr, + } => { + let vector_rows = + crate::vector_index::rows_for_entity_state(&vectors, &status, &fields); + match store .backfill_entity_vectors( tenant.as_str(), - entity_type, + &entity_type, entity_id, + reconciliation_generation, + sequence_nr, &vector_rows, ) .await { - Ok(()) => newly_indexed += 1, - Err(e) => { + Ok(()) if vector_rows.is_empty() => empty += 1, + Ok(()) => indexed += 1, + Err(error) => { failed += 1; tracing::warn!( - error = %e, entity_type = %entity_type, entity_id = %entity_id, - "vector index backfill: upsert failed" + error = %error, + entity_type = %entity_type, + entity_id = %entity_id, + sequence_nr, + "vector index backfill: reconciliation failed" ); } } } - EntityLoadOutcome::Skip => { - // A deleted (or phantom) entity must hold no vector rows — purge - // any it still has so a soft-deleted entity is never ranked - // (reconcile with an empty row set). Harmless when there is nothing - // to purge. - if let Err(e) = store - .backfill_entity_vectors(tenant.as_str(), entity_type, entity_id, &[]) + EntityLoadOutcome::Skip { sequence_nr } => { + if let Err(error) = store + .backfill_entity_vectors( + tenant.as_str(), + &entity_type, + entity_id, + reconciliation_generation, + sequence_nr, + &[], + ) .await { failed += 1; tracing::warn!( - error = %e, entity_type = %entity_type, entity_id = %entity_id, - "vector index backfill: purge of deleted/phantom entity failed" + error = %error, + entity_type = %entity_type, + entity_id = %entity_id, + sequence_nr, + "vector index backfill: purge reconciliation failed" ); } else { - skipped += 1; + empty += 1; } } EntityLoadOutcome::LoadFailed => { failed += 1; tracing::warn!( - entity_type = %entity_type, entity_id = %entity_id, - "vector index backfill: existing entity could not be loaded; type will NOT be watermarked" + entity_type = %entity_type, + entity_id = %entity_id, + "vector index backfill: durable stream could not be loaded; type will not be watermarked" ); } } tokio::task::yield_now().await; } - // Watermark only if nothing failed — every existing entity was indexed or is - // definitively skippable. Otherwise a later run resumes from the remainder. - if failed == 0 { - if let Some((store, _)) = state.event_journal() - && let Err(e) = store - .mark_vector_index_backfilled(tenant.as_str(), entity_type, ¤t_set) - .await - { - tracing::error!( - tenant = %tenant, entity_type = %entity_type, error = %e, - "vector index backfill: failed to persist watermark" - ); - } - tracing::info!( - tenant = %tenant, entity_type = %entity_type, vector_set = %current_set, - total, newly_indexed, already, skipped, - "entity_vector_index backfill complete; type watermarked" - ); - } else { + if failed != 0 { tracing::warn!( - tenant = %tenant, entity_type = %entity_type, - total, newly_indexed, already, skipped, failed, - "vector index backfill: {failed} entities unresolved; type NOT watermarked (will resume next run)" + tenant = %tenant, + entity_type = %entity_type, + total, + indexed, + empty, + failed, + "vector index backfill incomplete; type not watermarked" ); + continue; + } + + match store + .mark_vector_index_backfilled( + tenant.as_str(), + &entity_type, + reconciliation_generation, + ¤t_set, + ) + .await + { + Ok(()) => tracing::info!( + tenant = %tenant, + entity_type = %entity_type, + vector_set = %current_set, + total, + indexed, + empty, + "entity_vector_index reconciliation complete; type watermarked" + ), + Err(error) => tracing::error!( + tenant = %tenant, + entity_type = %entity_type, + error = %error, + "vector index backfill converged but watermark persistence failed" + ), } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn previously_watermarked_empty_vector_type_remains_in_work_set() { + let current_types = BTreeSet::from(["Item".to_string()]); + let covered = BTreeMap::from([ + ( + "Item".to_string(), + "v2|embed:vector:model:2:cosine".to_string(), + ), + ("Legacy".to_string(), "v1|embed".to_string()), + ]); + + assert_eq!( + vector_backfill_work_types(¤t_types, &covered, &BTreeSet::new()), + BTreeSet::from(["Item".to_string(), "Legacy".to_string()]) + ); + } + + #[test] + fn interrupted_empty_reconciliation_remains_in_work_set_without_a_watermark() { + let current_types = BTreeSet::from(["Item".to_string()]); + let covered = BTreeMap::new(); + let reconciliation_types = BTreeSet::from(["Item".to_string()]); + + assert_eq!( + vector_backfill_work_types(¤t_types, &covered, &reconciliation_types), + BTreeSet::from(["Item".to_string()]) + ); + } +} diff --git a/crates/temper-server/src/storage/data_only_create.rs b/crates/temper-server/src/storage/data_only_create.rs new file mode 100644 index 000000000..f76c215c6 --- /dev/null +++ b/crates/temper-server/src/storage/data_only_create.rs @@ -0,0 +1,38 @@ +use temper_runtime::persistence::{PersistenceEnvelope, PersistenceError}; + +/// Inputs for a native brand-new data-only entity create. +/// +/// This capability is only valid for entities whose first durable event and +/// first query projection row can be inserted atomically by a storage backend. +pub struct DataOnlyCreateRecord<'a> { + /// Tenant that owns the entity. + pub tenant: &'a str, + /// Entity type being created. + pub entity_type: &'a str, + /// Entity id being created. + pub entity_id: &'a str, + /// Initial entity status. + pub status: &'a str, + /// Projection fields to store in the query catalog and scalar index. + pub fields: &'a serde_json::Value, + /// Full response projection to store in the query catalog. + pub state: &'a serde_json::Value, + /// First event envelope to append at sequence number 1. + pub event: &'a PersistenceEnvelope, + /// Fingerprint of the exact table snapshot that derived the event. + pub spec_declaration_fingerprint: Option<&'a str>, +} + +/// Optional native storage capability for brand-new data-only creates. +#[async_trait::async_trait] +pub trait DataOnlyCreateStore: Send + Sync { + /// Persist the first event and initial projection atomically. + /// + /// Returns the new sequence number on success. Duplicate first events or + /// duplicate projection rows should return [`PersistenceError::ConcurrencyViolation`] + /// so the caller can decline the fast path and use the generic path. + async fn create_data_only_entity( + &self, + record: DataOnlyCreateRecord<'_>, + ) -> Result; +} diff --git a/crates/temper-server/src/storage/mod.rs b/crates/temper-server/src/storage/mod.rs index f689b7db0..b798a7543 100644 --- a/crates/temper-server/src/storage/mod.rs +++ b/crates/temper-server/src/storage/mod.rs @@ -34,12 +34,16 @@ use crate::platform_store::PlatformStore; use crate::platform_store::SimPlatformStore; use crate::state::trajectory::{TrajectoryEntry, TrajectorySource}; +mod data_only_create; mod published_artifacts; mod query_plane_impls; mod query_plane_read; +mod vector_event_store; +pub use data_only_create::{DataOnlyCreateRecord, DataOnlyCreateStore}; pub use published_artifacts::{ PublishedArtifactStore, PublishedArtifactStoreRow, PublishedArtifactStoreUpsert, }; +pub use vector_event_store::AppendIndexRows; mod query_plane; pub use query_plane::{ EntityCatalogRow, QueryFieldIndexOrder, QueryFieldIndexOrderDirection, QueryFieldIndexPage, @@ -84,19 +88,40 @@ pub trait DynEventStore: Send + Sync { persistence_id: &'a str, expected_sequence: u64, events: &'a [PersistenceEnvelope], - key_rows: &'a [temper_runtime::persistence::EntityKeyRow], - vector_rows: &'a [temper_runtime::persistence::EntityVectorRow], - reconcile_vectors: bool, + index_rows: AppendIndexRows<'a>, + ) -> EventStoreFuture<'a, Result>; + + fn persist_spec_declaration<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + declaration_fingerprint: &'a str, ) -> EventStoreFuture<'a, Result>; + fn spec_declaration_entity_types<'a>( + &'a self, + tenant: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>>; + fn backfill_entity_vectors<'a>( &'a self, tenant: &'a str, entity_type: &'a str, entity_id: &'a str, + reconciliation_generation: u64, + observed_sequence: u64, vector_rows: &'a [temper_runtime::persistence::EntityVectorRow], ) -> EventStoreFuture<'a, Result<(), PersistenceError>>; + fn begin_vector_index_reconciliation<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + vector_set: &'a str, + declaration_revision: u64, + declaration_fingerprint: &'a str, + ) -> EventStoreFuture<'a, Result>; + fn vector_candidates<'a>( &'a self, tenant: &'a str, @@ -113,6 +138,7 @@ pub trait DynEventStore: Send + Sync { &'a self, tenant: &'a str, entity_type: &'a str, + reconciliation_generation: u64, vector_set: &'a str, ) -> EventStoreFuture<'a, Result<(), PersistenceError>>; @@ -121,6 +147,11 @@ pub trait DynEventStore: Send + Sync { tenant: &'a str, ) -> EventStoreFuture<'a, Result, PersistenceError>>; + fn vector_reconciliation_entity_types<'a>( + &'a self, + tenant: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>>; + fn vectored_entity_ids_for_type<'a>( &'a self, tenant: &'a str, @@ -184,6 +215,12 @@ pub trait DynEventStore: Send + Sync { entity_type: &'a str, ) -> EventStoreFuture<'a, Result, PersistenceError>>; + fn list_vector_repair_entity_ids<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>>; + fn list_entity_ids_limited<'a>( &'a self, tenant: &'a str, @@ -246,26 +283,48 @@ where persistence_id: &'a str, expected_sequence: u64, events: &'a [PersistenceEnvelope], - key_rows: &'a [temper_runtime::persistence::EntityKeyRow], - vector_rows: &'a [temper_runtime::persistence::EntityVectorRow], - reconcile_vectors: bool, + index_rows: AppendIndexRows<'a>, ) -> EventStoreFuture<'a, Result> { Box::pin(EventStore::append_with_index_rows( self, persistence_id, expected_sequence, events, - key_rows, - vector_rows, - reconcile_vectors, + index_rows.key_rows, + index_rows.vector_rows, + index_rows.reconcile_vectors, + index_rows.spec_declaration_fingerprint, + )) + } + + fn persist_spec_declaration<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + declaration_fingerprint: &'a str, + ) -> EventStoreFuture<'a, Result> { + Box::pin(EventStore::persist_spec_declaration( + self, + tenant, + entity_type, + declaration_fingerprint, )) } + fn spec_declaration_entity_types<'a>( + &'a self, + tenant: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::spec_declaration_entity_types(self, tenant)) + } + fn backfill_entity_vectors<'a>( &'a self, tenant: &'a str, entity_type: &'a str, entity_id: &'a str, + reconciliation_generation: u64, + observed_sequence: u64, vector_rows: &'a [temper_runtime::persistence::EntityVectorRow], ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { Box::pin(EventStore::backfill_entity_vectors( @@ -273,10 +332,30 @@ where tenant, entity_type, entity_id, + reconciliation_generation, + observed_sequence, vector_rows, )) } + fn begin_vector_index_reconciliation<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + vector_set: &'a str, + declaration_revision: u64, + declaration_fingerprint: &'a str, + ) -> EventStoreFuture<'a, Result> { + Box::pin(EventStore::begin_vector_index_reconciliation( + self, + tenant, + entity_type, + vector_set, + declaration_revision, + declaration_fingerprint, + )) + } + fn vector_candidates<'a>( &'a self, tenant: &'a str, @@ -302,12 +381,14 @@ where &'a self, tenant: &'a str, entity_type: &'a str, + reconciliation_generation: u64, vector_set: &'a str, ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { Box::pin(EventStore::mark_vector_index_backfilled( self, tenant, entity_type, + reconciliation_generation, vector_set, )) } @@ -319,6 +400,13 @@ where Box::pin(EventStore::vector_index_backfilled_types(self, tenant)) } + fn vector_reconciliation_entity_types<'a>( + &'a self, + tenant: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::vector_reconciliation_entity_types(self, tenant)) + } + fn vectored_entity_ids_for_type<'a>( &'a self, tenant: &'a str, @@ -436,6 +524,18 @@ where )) } + fn list_vector_repair_entity_ids<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::list_vector_repair_entity_ids( + self, + tenant, + entity_type, + )) + } + fn list_entity_ids_limited<'a>( &'a self, tenant: &'a str, @@ -512,80 +612,6 @@ impl BoxedEventStore { .await } - pub async fn append_with_index_rows( - &self, - persistence_id: &str, - expected_sequence: u64, - events: &[PersistenceEnvelope], - key_rows: &[temper_runtime::persistence::EntityKeyRow], - vector_rows: &[temper_runtime::persistence::EntityVectorRow], - reconcile_vectors: bool, - ) -> Result { - self.0 - .append_with_index_rows( - persistence_id, - expected_sequence, - events, - key_rows, - vector_rows, - reconcile_vectors, - ) - .await - } - - pub async fn backfill_entity_vectors( - &self, - tenant: &str, - entity_type: &str, - entity_id: &str, - vector_rows: &[temper_runtime::persistence::EntityVectorRow], - ) -> Result<(), PersistenceError> { - self.0 - .backfill_entity_vectors(tenant, entity_type, entity_id, vector_rows) - .await - } - - pub async fn vector_candidates( - &self, - tenant: &str, - entity_type: &str, - decl_name: &str, - model_tag: &str, - limit: usize, - ) -> Result, PersistenceError> { - self.0 - .vector_candidates(tenant, entity_type, decl_name, model_tag, limit) - .await - } - - pub async fn mark_vector_index_backfilled( - &self, - tenant: &str, - entity_type: &str, - vector_set: &str, - ) -> Result<(), PersistenceError> { - self.0 - .mark_vector_index_backfilled(tenant, entity_type, vector_set) - .await - } - - pub async fn vector_index_backfilled_types( - &self, - tenant: &str, - ) -> Result, PersistenceError> { - self.0.vector_index_backfilled_types(tenant).await - } - - pub async fn vectored_entity_ids_for_type( - &self, - tenant: &str, - entity_type: &str, - ) -> Result, PersistenceError> { - self.0 - .vectored_entity_ids_for_type(tenant, entity_type) - .await - } - pub async fn lookup_by_key( &self, tenant: &str, @@ -743,41 +769,6 @@ impl From for PolicyStoreRow { } } -/// Inputs for a native brand-new data-only entity create. -/// -/// This capability is only valid for entities whose first durable event and -/// first query projection row can be inserted atomically by a storage backend. -pub struct DataOnlyCreateRecord<'a> { - /// Tenant that owns the entity. - pub tenant: &'a str, - /// Entity type being created. - pub entity_type: &'a str, - /// Entity id being created. - pub entity_id: &'a str, - /// Initial entity status. - pub status: &'a str, - /// Projection fields to store in the query catalog and scalar index. - pub fields: &'a serde_json::Value, - /// Full response projection to store in the query catalog. - pub state: &'a serde_json::Value, - /// First event envelope to append at sequence number 1. - pub event: &'a PersistenceEnvelope, -} - -/// Optional native storage capability for brand-new data-only creates. -#[async_trait::async_trait] -pub trait DataOnlyCreateStore: Send + Sync { - /// Persist the first event and initial projection atomically. - /// - /// Returns the new sequence number on success. Duplicate first events or - /// duplicate projection rows should return [`PersistenceError::ConcurrencyViolation`] - /// so the caller can decline the fast path and use the generic path. - async fn create_data_only_entity( - &self, - record: DataOnlyCreateRecord<'_>, - ) -> Result; -} - /// Durable observe trajectory sink. #[async_trait::async_trait] pub trait TrajectorySink: Send + Sync { @@ -2672,6 +2663,7 @@ impl DataOnlyCreateStore for PostgresEventStore { record.fields, record.state, record.event, + record.spec_declaration_fingerprint, ) .await } diff --git a/crates/temper-server/src/storage/vector_event_store.rs b/crates/temper-server/src/storage/vector_event_store.rs new file mode 100644 index 000000000..5ee54c7d5 --- /dev/null +++ b/crates/temper-server/src/storage/vector_event_store.rs @@ -0,0 +1,164 @@ +use temper_runtime::persistence::{ + EntityVectorCandidate, EntityVectorRow, PersistenceEnvelope, PersistenceError, +}; + +use super::BoxedEventStore; + +/// Derived index rows and declaration authority co-committed with one append. +pub struct AppendIndexRows<'a> { + /// Unique-key rows derived from the exact transition-table snapshot. + pub key_rows: &'a [temper_runtime::persistence::EntityKeyRow], + /// Vector rows derived from the exact transition-table snapshot. + pub vector_rows: &'a [temper_runtime::persistence::EntityVectorRow], + /// Whether vector rows absent from this append must be removed. + pub reconcile_vectors: bool, + /// Fingerprint of the exact declaration snapshot used by the writer. + pub spec_declaration_fingerprint: Option<&'a str>, +} + +impl BoxedEventStore { + /// Append journal events and co-commit their derived key/vector index rows. + pub async fn append_with_index_rows( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + index_rows: AppendIndexRows<'_>, + ) -> Result { + self.0 + .append_with_index_rows(persistence_id, expected_sequence, events, index_rows) + .await + } + + /// Persist one declaration fingerprint or absence tombstone. + pub async fn persist_spec_declaration( + &self, + tenant: &str, + entity_type: &str, + declaration_fingerprint: &str, + ) -> Result { + self.0 + .persist_spec_declaration(tenant, entity_type, declaration_fingerprint) + .await + } + + /// Return currently present durable declaration types for one tenant. + pub async fn spec_declaration_entity_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + self.0.spec_declaration_entity_types(tenant).await + } + + /// Replace one entity's vector rows behind a generation and sequence fence. + pub async fn backfill_entity_vectors( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + reconciliation_generation: u64, + observed_sequence: u64, + vector_rows: &[EntityVectorRow], + ) -> Result<(), PersistenceError> { + self.0 + .backfill_entity_vectors( + tenant, + entity_type, + entity_id, + reconciliation_generation, + observed_sequence, + vector_rows, + ) + .await + } + + /// Claim the durable generation for one declaration snapshot. + pub async fn begin_vector_index_reconciliation( + &self, + tenant: &str, + entity_type: &str, + vector_set: &str, + declaration_revision: u64, + declaration_fingerprint: &str, + ) -> Result { + self.0 + .begin_vector_index_reconciliation( + tenant, + entity_type, + vector_set, + declaration_revision, + declaration_fingerprint, + ) + .await + } + + /// Read bounded candidates from one declaration/model partition. + pub async fn vector_candidates( + &self, + tenant: &str, + entity_type: &str, + decl_name: &str, + model_tag: &str, + limit: usize, + ) -> Result, PersistenceError> { + self.0 + .vector_candidates(tenant, entity_type, decl_name, model_tag, limit) + .await + } + + /// Publish a generation-checked completion watermark for an entity type. + pub async fn mark_vector_index_backfilled( + &self, + tenant: &str, + entity_type: &str, + reconciliation_generation: u64, + vector_set: &str, + ) -> Result<(), PersistenceError> { + self.0 + .mark_vector_index_backfilled( + tenant, + entity_type, + reconciliation_generation, + vector_set, + ) + .await + } + + /// List entity types and declaration signatures with completion watermarks. + pub async fn vector_index_backfilled_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + self.0.vector_index_backfilled_types(tenant).await + } + + /// List entity types with any durable vector-reconciliation state. + pub async fn vector_reconciliation_entity_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + self.0.vector_reconciliation_entity_types(tenant).await + } + + /// List entity IDs that currently retain vector candidates for a type. + pub async fn vectored_entity_ids_for_type( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + self.0 + .vectored_entity_ids_for_type(tenant, entity_type) + .await + } + + /// List all journaled IDs, including deleted streams, for vector repair. + pub async fn list_vector_repair_entity_ids( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + self.0 + .list_vector_repair_entity_ids(tenant, entity_type) + .await + } +} diff --git a/crates/temper-server/src/vector_index.rs b/crates/temper-server/src/vector_index.rs index ddc2d6098..5155904fe 100644 --- a/crates/temper-server/src/vector_index.rs +++ b/crates/temper-server/src/vector_index.rs @@ -11,24 +11,26 @@ //! id. This is what makes kernel-side similarity admissible under deterministic //! simulation where app-side similarity never was. -use temper_runtime::persistence::EntityVectorCandidate; +use temper_runtime::persistence::{EntityVectorCandidate, EntityVectorRow}; // The blob encoders live beside `EntityVectorRow` in temper-runtime so every store // and the kernel ranking share one byte layout; re-exported here for callers that // reach for them through the vector-index module. pub use temper_runtime::persistence::{pack_f32_le, unpack_f32_le}; -/// The stable signature of a type's declared vector-path set (ADR-0155): each path -/// rendered as `name:property:model_property:dims:metric`, sorted by name and -/// semicolon-joined. Recorded in the vector-index backfill watermark and compared -/// on the next backfill, so ANY change — a new path, or an in-place edit to a -/// path's property/model_property/dims/metric — changes the signature and re-indexes -/// the type instead of being treated as already complete. Including `dims` matters: -/// an edited `dims` makes every existing row the wrong length (they would be dropped -/// at read time as corrupt), so the type must be re-embedded/reconciled. Deterministic +/// The stable, protocol-revisioned signature of a type's declared vector-path set +/// (ADR-0155/ADR-0181): each path is rendered as +/// `name:property:model_property:dims:metric`, sorted by name, and semicolon-joined. +/// Recorded in the vector-index backfill watermark and compared on the next +/// backfill, so ANY declaration change re-indexes the type. The protocol prefix +/// deliberately invalidates pre-ADR-0181 watermarks once, forcing every legacy row +/// through sequence-aware reconciliation. Including `dims` matters: an edited +/// `dims` makes every existing row the wrong length (they would be dropped at read +/// time as corrupt), so the type must be re-embedded/reconciled. Deterministic /// (sorted, no map iteration). Mirrors `declared_key_set_signature`. pub fn declared_vector_set_signature( vectors: &[temper_jit::table::types::DeclaredVector], ) -> String { + const RECONCILIATION_PROTOCOL_REVISION: &str = "v2"; let mut entries: Vec = vectors .iter() .map(|v| { @@ -39,7 +41,7 @@ pub fn declared_vector_set_signature( }) .collect(); entries.sort(); - entries.join(";") + format!("{RECONCILIATION_PROTOCOL_REVISION}|{}", entries.join(";")) } /// The similarity metric declared on a `[[vector]]` path. @@ -99,6 +101,43 @@ pub fn parse_vector_property(value: &serde_json::Value, dims: usize) -> Option Vec { + if status == "Deleted" { + return Vec::new(); + } + let Some(field_map) = fields.as_object() else { + return Vec::new(); + }; + vectors + .iter() + .filter_map(|decl| { + let vector = field_map + .get(&decl.property) + .and_then(|value| parse_vector_property(value, decl.dims))?; + let model_tag = field_map + .get(&decl.model_property) + .and_then(|value| value.as_str()) + .filter(|tag| !tag.is_empty())?; + Some(EntityVectorRow { + decl_name: decl.name.clone(), + model_tag: model_tag.to_string(), + vector, + }) + }) + .collect() +} + /// One ranked entity plus its closeness score (higher = nearer). #[derive(Debug, Clone, PartialEq)] pub struct ScoredEntity { @@ -213,6 +252,29 @@ mod tests { } } + #[test] + fn empty_vector_set_signature_is_protocol_revisioned() { + assert_eq!(declared_vector_set_signature(&[]), "v2|"); + } + + #[test] + fn remove_all_then_readd_identical_vector_changes_watermark_each_time() { + let declaration = temper_jit::table::types::DeclaredVector { + name: "embed".to_string(), + property: "vector".to_string(), + model_property: "model".to_string(), + dims: 2, + metric: "cosine".to_string(), + }; + let declared = declared_vector_set_signature(std::slice::from_ref(&declaration)); + let removed = declared_vector_set_signature(&[]); + let readded = declared_vector_set_signature(&[declaration]); + + assert_ne!(declared, removed); + assert_eq!(declared, readded); + assert_ne!(removed, readded); + } + #[test] fn pack_unpack_roundtrips() { let v = vec![0.0f32, 1.5, -2.25, 384.0]; diff --git a/crates/temper-server/tests/common/platform_harness.rs b/crates/temper-server/tests/common/platform_harness.rs index 48edbd9bb..e78bb47e5 100644 --- a/crates/temper-server/tests/common/platform_harness.rs +++ b/crates/temper-server/tests/common/platform_harness.rs @@ -94,7 +94,18 @@ impl SimPlatformHarness { pub fn register_inline_spec(&self, tenant: &str, entity_type: &str, ioa_source: &str) { let automaton = temper_spec::automaton::parse_automaton(ioa_source).expect("inline IOA should parse"); - let table = temper_jit::table::TransitionTable::from_automaton(&automaton); + let table = temper_jit::table::TransitionTable::from_ioa_source(ioa_source); + let fingerprint = table + .spec_declaration_fingerprint + .as_deref() + .expect("inline IOA table should carry its exact declaration fingerprint"); + let revision = + self.sim_event_store + .persist_spec_declaration(tenant, entity_type, fingerprint); + assert!( + revision > 0, + "inline hot-swap must advance durable declaration authority first" + ); let mut registry = self.platform_state.server.registry.write().unwrap(); // ci-ok: infallible lock let spec = registry .get_spec_mut(&TenantId::new(tenant), entity_type) diff --git a/crates/temper-server/tests/dst_entity_vector_index.rs b/crates/temper-server/tests/dst_entity_vector_index.rs index f26baa214..9c4f6f7a3 100644 --- a/crates/temper-server/tests/dst_entity_vector_index.rs +++ b/crates/temper-server/tests/dst_entity_vector_index.rs @@ -15,8 +15,9 @@ use std::time::Duration; use temper_jit::table::TransitionTable; use temper_runtime::ActorSystem; -use temper_runtime::scheduler::install_deterministic_context; -use temper_server::storage::{BackendLabel, BoxedEventStore}; +use temper_runtime::persistence::{EntityVectorRow, EventMetadata, PersistenceEnvelope}; +use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; +use temper_server::storage::{AppendIndexRows, BackendLabel, BoxedEventStore}; use temper_server::vector_index::{VectorMetric, rank_nearest}; use temper_server::{EntityActor, EntityMsg, EntityResponse}; use temper_store_sim::SimEventStore; @@ -55,7 +56,7 @@ async fn create_item( entity_id: &str, embedding: &[f32], model: &str, -) { +) -> temper_runtime::actor::ActorRef { let actor = EntityActor::with_persistence( "Item", entity_id, @@ -74,6 +75,7 @@ async fn create_item( ) .await; assert!(r.success, "Create failed: {:?}", r.error); + actor_ref } /// The fixed corpus every seed writes. Cosine nearest to [1,0,0,0] is `a` @@ -90,6 +92,280 @@ fn corpus() -> Vec<(&'static str, [f32; 4], &'static str)> { ] } +fn test_envelope(event_type: &str) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: 0, + event_type: event_type.to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: "dst-vector-reconcile".to_string(), + }, + } +} + +/// A repair observed at N must not overwrite live N+1, and a repair observed at +/// N+1 must not resurrect vectors purged by live N+2. Explicit deterministic +/// schedule through the server's dynamic EventStore path, all seeds. +#[tokio::test] +async fn dst_delayed_vector_repair_is_sequence_monotonic() { + for seed in 0..NUM_SEEDS { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let sim_store = SimEventStore::no_faults(seed); + sim_store.persist_spec_declaration("default", "Item", "rev-1"); + let store = BoxedEventStore::new(sim_store); + let generation = store + .begin_vector_index_reconciliation("default", "Item", "v2|embed", 1, "rev-1") + .await + .expect("begin vector reconciliation generation"); + let persistence_id = format!("default:Item:item-race-{seed}"); + let entity_id = format!("item-race-{seed}"); + let stale_row = EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0, 0.0, 0.0], + }; + let live_row = EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![0.0, 1.0, 0.0, 0.0], + }; + + store + .append_with_index_rows( + &persistence_id, + 0, + &[test_envelope("Created")], + AppendIndexRows { + key_rows: &[], + vector_rows: std::slice::from_ref(&stale_row), + reconcile_vectors: true, + spec_declaration_fingerprint: Some("rev-1"), + }, + ) + .await + .expect("append sequence 1"); + store + .append_with_index_rows( + &persistence_id, + 1, + &[test_envelope("Updated")], + AppendIndexRows { + key_rows: &[], + vector_rows: std::slice::from_ref(&live_row), + reconcile_vectors: true, + spec_declaration_fingerprint: Some("rev-1"), + }, + ) + .await + .expect("append sequence 2"); + store + .backfill_entity_vectors( + "default", + "Item", + &entity_id, + generation, + 1, + std::slice::from_ref(&stale_row), + ) + .await + .expect("ignore delayed sequence-1 repair"); + assert_eq!( + store + .vector_candidates("default", "Item", "embed", "m1", 10) + .await + .expect("read live candidate")[0] + .vector, + live_row.vector.clone(), + "seed {seed}: delayed sequence 1 must not overwrite live sequence 2" + ); + + store + .append_with_index_rows( + &persistence_id, + 2, + &[test_envelope("Deleted")], + AppendIndexRows { + key_rows: &[], + vector_rows: &[], + reconcile_vectors: true, + spec_declaration_fingerprint: Some("rev-1"), + }, + ) + .await + .expect("append sequence-3 purge"); + store + .backfill_entity_vectors( + "default", + "Item", + &entity_id, + generation, + 2, + std::slice::from_ref(&live_row), + ) + .await + .expect("ignore delayed sequence-2 repair"); + assert!( + store + .vector_candidates("default", "Item", "embed", "m1", 10) + .await + .expect("read purged partition") + .is_empty(), + "seed {seed}: delayed sequence 2 must not resurrect the sequence-3 purge" + ); + } +} + +/// Declaration authority follows its monotonic revision, never coordinator +/// arrival order. A stale A replica cannot supersede completed B, while a later +/// authoritative A revision remains a valid re-add. +#[tokio::test(flavor = "current_thread")] +async fn dst_stale_declaration_revision_cannot_reclaim_generation() { + for seed in 0..NUM_SEEDS { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let sim_store = SimEventStore::no_faults(seed); + sim_store.persist_spec_declaration("default", "Item", "rev-a"); + let store = BoxedEventStore::new(sim_store.clone()); + let generation_a = store + .begin_vector_index_reconciliation("default", "Item", "v2|a", 1, "rev-a") + .await + .expect("begin declaration A"); + store + .mark_vector_index_backfilled("default", "Item", generation_a, "v2|a") + .await + .expect("publish declaration A"); + + sim_store.persist_spec_declaration("default", "Item", "rev-b"); + let generation_b = store + .begin_vector_index_reconciliation("default", "Item", "v2|b", 2, "rev-b") + .await + .expect("begin declaration B"); + store + .mark_vector_index_backfilled("default", "Item", generation_b, "v2|b") + .await + .expect("publish declaration B"); + + assert!( + store + .begin_vector_index_reconciliation("default", "Item", "v2|a", u64::MAX, "rev-a",) + .await + .is_err(), + "seed {seed}: stale A must not reclaim authority after B" + ); + assert_eq!( + store + .vector_index_backfilled_types("default") + .await + .expect("read B watermark"), + vec![("Item".to_string(), "v2|b".to_string())], + "seed {seed}: stale A must leave B's completion claim intact" + ); + + sim_store.persist_spec_declaration("default", "Item", "rev-a"); + let readded_a = store + .begin_vector_index_reconciliation("default", "Item", "v2|a", 3, "rev-a") + .await + .expect("begin later authoritative A re-add"); + assert!(readded_a > generation_b); + + // Crash after allocating the remove-all generation but before publishing + // its empty watermark. A restarted server-side dynamic store handle must + // resume that generation, then fence it when the identical spec is re-added. + sim_store.persist_spec_declaration("default", "Item", "absent:v1"); + let absent_generation = store + .begin_vector_index_reconciliation("default", "Item", "v2|", 4, "absent:v1") + .await + .expect("begin declaration tombstone"); + let restarted_store = store.clone(); + let resumed_generation = restarted_store + .begin_vector_index_reconciliation("default", "Item", "v2|", 4, "absent:v1") + .await + .expect("resume declaration tombstone after restart"); + assert_eq!(resumed_generation, absent_generation, "seed {seed}"); + restarted_store + .mark_vector_index_backfilled("default", "Item", resumed_generation, "v2|") + .await + .expect("publish resumed empty declaration"); + sim_store.persist_spec_declaration("default", "Item", "rev-a"); + let post_restart_readd = restarted_store + .begin_vector_index_reconciliation("default", "Item", "v2|a", 5, "rev-a") + .await + .expect("re-add declaration after resumed deletion"); + assert!(post_restart_readd > resumed_generation, "seed {seed}"); + } +} + +/// The direct/OData delete message persists before mutating the actor's in-memory +/// status. Vector derivation must use the event's post-transition status so the +/// journal delete and empty candidate set share one atomic append. +#[tokio::test] +async fn dst_direct_delete_co_commits_vector_purge_before_delayed_repair() { + for seed in 0..NUM_SEEDS { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let fingerprint = temper_store_turso::spec_content_hash(ITEM_IOA); + let sim_store = SimEventStore::no_faults(seed); + sim_store.persist_spec_declaration("default", "Item", &fingerprint); + let store = BoxedEventStore::new(sim_store); + let generation = store + .begin_vector_index_reconciliation("default", "Item", "v2|embed", 1, &fingerprint) + .await + .expect("begin vector reconciliation generation"); + let table = item_table(); + let system = ActorSystem::new("dst-vector-direct-delete"); + let entity_id = format!("item-delete-{seed}"); + let persistence_id = format!("default:Item:{entity_id}"); + let actor_ref = create_item( + &system, + &table, + &store, + &entity_id, + &[1.0, 0.0, 0.0, 0.0], + "m1", + ) + .await; + let observed_sequence = store + .read_events(&persistence_id, 0) + .await + .expect("read pre-delete journal") + .last() + .expect("Create event exists") + .sequence_nr; + let stale_row = EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0, 0.0, 0.0], + }; + + let deleted: EntityResponse = actor_ref + .ask(EntityMsg::Delete, Duration::from_secs(5)) + .await + .expect("actor should respond to direct delete"); + assert!(deleted.success, "seed {seed}: direct delete failed"); + store + .backfill_entity_vectors( + "default", + "Item", + &entity_id, + generation, + observed_sequence, + std::slice::from_ref(&stale_row), + ) + .await + .expect("delayed pre-delete repair is a successful no-op"); + assert!( + store + .vector_candidates("default", "Item", "embed", "m1", 10) + .await + .expect("read deleted vector partition") + .is_empty(), + "seed {seed}: direct delete must purge candidates and reject delayed repair" + ); + } +} + #[tokio::test] async fn dst_nearest_ranking_is_reproducible_across_seeds() { let query = [1.0f32, 0.0, 0.0, 0.0]; @@ -97,7 +373,13 @@ async fn dst_nearest_ranking_is_reproducible_across_seeds() { for seed in 0..NUM_SEEDS { let (_guard, _clock, _id) = install_deterministic_context(seed); - let store: BoxedEventStore = BoxedEventStore::new(SimEventStore::no_faults(seed)); + let sim_store = SimEventStore::no_faults(seed); + sim_store.persist_spec_declaration( + "default", + "Item", + &temper_store_turso::spec_content_hash(ITEM_IOA), + ); + let store: BoxedEventStore = BoxedEventStore::new(sim_store); let table = item_table(); let system = ActorSystem::new("dst-vector"); diff --git a/crates/temper-server/tests/dst_hotswap.rs b/crates/temper-server/tests/dst_hotswap.rs index 247f484b9..05807ce51 100644 --- a/crates/temper-server/tests/dst_hotswap.rs +++ b/crates/temper-server/tests/dst_hotswap.rs @@ -8,7 +8,9 @@ mod common; use temper_runtime::scheduler::install_deterministic_context; use temper_runtime::tenant::TenantId; +use temper_server::ServerState; use temper_spec::csdl::parse_csdl; +use temper_store_sim::SimEventStore; /// An extended Order spec with an additional "Archived" state and "ArchiveOrder" action. const ORDER_V2_IOA: &str = r#" @@ -75,6 +77,23 @@ to = "Archived" kind = "input" "#; +fn publish_order_v2(state: &ServerState, sim_store: &SimEventStore) { + let fingerprint = temper_store_turso::spec_content_hash(ORDER_V2_IOA); + let revision = sim_store.persist_spec_declaration("default", "Order", &fingerprint); + assert!( + revision > 0, + "hot-swap must advance durable declaration authority first" + ); + let mut registry = state.registry.write().expect("registry lock"); // ci-ok: infallible lock + let csdl = parse_csdl(common::CSDL_XML).expect("CSDL parse"); + registry.register_tenant( + "default", + csdl, + common::CSDL_XML.to_string(), + &[("Order", ORDER_V2_IOA)], + ); +} + // ========================================================================= // Test: Hot-swap adds new states visible to live entities // ========================================================================= @@ -83,7 +102,7 @@ kind = "input" async fn dst_hotswap_entity_sees_new_table() { for seed in 0..50 { let (_guard, _clock, _id_gen) = install_deterministic_context(seed); - let (state, _sim_store) = common::build_default_state(seed, "dst-hotswap"); + let (state, sim_store) = common::build_default_state(seed, "dst-hotswap"); let tenant = TenantId::default(); // Create an Order and advance to Confirmed. @@ -122,16 +141,7 @@ async fn dst_hotswap_entity_sees_new_table() { assert_eq!(r.state.status, "Confirmed"); // Hot-swap to v2 spec (adds "Archived" state and "ArchiveOrder" action). - { - let mut reg = state.registry.write().expect("registry lock"); // ci-ok: infallible lock - let csdl = parse_csdl(common::CSDL_XML).expect("CSDL parse"); - reg.register_tenant( - "default", - csdl, - common::CSDL_XML.to_string(), - &[("Order", ORDER_V2_IOA)], - ); - } + publish_order_v2(&state, &sim_store); // Advance through the remaining states using v2 table. for action in &["ProcessOrder", "ShipOrder", "DeliverOrder"] { @@ -175,7 +185,7 @@ async fn dst_hotswap_entity_sees_new_table() { #[tokio::test] async fn dst_hotswap_version_increases() { let (_guard, _clock, _id_gen) = install_deterministic_context(42); - let (state, _sim_store) = common::build_default_state(42, "dst-hotswap"); + let (state, sim_store) = common::build_default_state(42, "dst-hotswap"); let tenant = TenantId::default(); // Get initial version. @@ -186,16 +196,7 @@ async fn dst_hotswap_version_increases() { }; // Hot-swap. - { - let mut reg = state.registry.write().expect("registry lock"); // ci-ok: infallible lock - let csdl = parse_csdl(common::CSDL_XML).expect("CSDL parse"); - reg.register_tenant( - "default", - csdl, - common::CSDL_XML.to_string(), - &[("Order", ORDER_V2_IOA)], - ); - } + publish_order_v2(&state, &sim_store); let v2 = { let reg = state.registry.read().expect("registry lock"); // ci-ok: infallible lock diff --git a/crates/temper-server/tests/dst_platform_boot.rs b/crates/temper-server/tests/dst_platform_boot.rs index 45a0f29fa..76abb6bc0 100644 --- a/crates/temper-server/tests/dst_platform_boot.rs +++ b/crates/temper-server/tests/dst_platform_boot.rs @@ -118,6 +118,12 @@ async fn dst_boot_cycle_with_store_faults() { .await; // Dispatch may fail due to injected write faults — that's expected. + // Fault injection targets the attempted operation. Recovery and the + // invariant audit must read the durable result without injecting a new + // truncation, or the audit would be measuring its own read fault rather + // than the state left by the failed/successful write. + let prev_event = harness.sim_event_store.disable_faults(); + // Restart — only successfully persisted state should be visible. harness.restart().await; @@ -129,6 +135,7 @@ async fn dst_boot_cycle_with_store_faults() { assert_data_invariants(&harness).await.unwrap_or_else(|e| { panic!("seed {seed}: data invariants failed after store faults: {e}") }); + harness.sim_event_store.restore_faults(prev_event); } } diff --git a/crates/temper-server/tests/dst_platform_rollback.rs b/crates/temper-server/tests/dst_platform_rollback.rs index becf7be2f..2d70aa050 100644 --- a/crates/temper-server/tests/dst_platform_rollback.rs +++ b/crates/temper-server/tests/dst_platform_rollback.rs @@ -9,7 +9,7 @@ mod common; use common::platform_harness::SimPlatformHarness; use common::platform_invariants::*; use temper_runtime::scheduler::install_deterministic_context; -use temper_server::platform_store::SimPlatformFaultConfig; +use temper_server::platform_store::{PlatformStore, SimPlatformFaultConfig}; use temper_store_sim::SimFaultConfig; const NUM_SEEDS: u64 = 50; @@ -47,6 +47,23 @@ async fn dst_rollback_install_failure_is_atomic() { Err(_) => { // Install failed — disable faults and verify no partial state. let prev = harness.sim_platform_store.disable_faults(); + let app_recorded = harness + .sim_platform_store + .is_app_installed("rollback-test", "project-management") + .await + .expect("read installed-app marker"); + if !app_recorded { + assert!( + harness + .sim_platform_store + .load_specs() + .await + .expect("read durable specs") + .iter() + .all(|row| row.tenant != "rollback-test"), + "seed {seed}: specs must remain unpublished when a pre-commit metadata write fails" + ); + } assert_p7_cedar_persistence(&harness) .await .unwrap_or_else(|e| panic!("seed {seed}: P7 failed after failed install: {e}")); @@ -101,6 +118,11 @@ async fn dst_rollback_dispatch_with_store_faults() { // Failures are expected — event store faults will cause some to fail. } + // Fault injection belongs to the attempted dispatches. Recovery and + // invariant reads must observe the durable outcome without injecting a + // second, unrelated truncation into the audit itself. + let prev_event = faulty_harness.sim_event_store.disable_faults(); + // Restart — only successfully persisted state should be visible. faulty_harness.restart().await; @@ -123,5 +145,6 @@ async fn dst_rollback_dispatch_with_store_faults() { ({success_count} succeeded): {e}" ) }); + faulty_harness.sim_event_store.restore_faults(prev_event); } } diff --git a/crates/temper-server/tests/dst_vector_reconciliation_restart.rs b/crates/temper-server/tests/dst_vector_reconciliation_restart.rs new file mode 100644 index 000000000..3b0de037d --- /dev/null +++ b/crates/temper-server/tests/dst_vector_reconciliation_restart.rs @@ -0,0 +1,341 @@ +//! DST: declaration deletion/restart/re-add preserves vector authority. + +use temper_jit::table::TransitionTable; +use temper_runtime::ActorSystem; +use temper_runtime::persistence::{ + EntityVectorRow, EventMetadata, EventStore, PersistenceEnvelope, +}; +use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; +use temper_runtime::tenant::TenantId; +use temper_server::entity_actor::EntityEvent; +use temper_server::registry::SpecRegistry; +use temper_server::vector_index::declared_vector_set_signature; +use temper_server::{ServerState, StorageStack}; +use temper_spec::csdl::parse_csdl; +use temper_store_sim::{SimEventStore, SimFaultConfig}; + +const ITEM_IOA: &str = include_str!("../../../test-fixtures/specs/vectored_item.ioa.toml"); +const ITEM_CSDL: &str = r#" + + + + + + + + + + + + + + + + +"#; + +fn registry_with_item(include_item: bool) -> SpecRegistry { + let mut registry = SpecRegistry::new(); + let csdl = parse_csdl(ITEM_CSDL).expect("parse Item CSDL"); + if include_item { + registry.register_tenant( + "default", + csdl, + ITEM_CSDL.to_string(), + &[("Item", ITEM_IOA)], + ); + } else { + registry.register_tenant("default", csdl, ITEM_CSDL.to_string(), &[]); + } + registry +} + +fn state_with_item(store: &SimEventStore, include_item: bool, seed: u64) -> ServerState { + let mut state = ServerState::from_registry( + ActorSystem::new(format!("dst-vector-restart-{seed}")), + registry_with_item(include_item), + ); + state.set_storage_stack(StorageStack::from_sim(store.clone(), None)); + state +} + +#[tokio::test(flavor = "current_thread")] +async fn deleted_vector_declaration_resumes_after_restart_and_readds_identically() { + // The 100-seed interleaving model lives in `dst_entity_vector_index`; this + // lifecycle integration adds the real ServerState teardown/reconstruction + // boundary once, because each state owns long-lived projection queues. + let seed = 0; + let (_guard, _clock, _id) = install_deterministic_context(seed); + let store = SimEventStore::no_faults(seed); + let tenant = TenantId::default(); + let fingerprint = temper_store_turso::spec_content_hash(ITEM_IOA); + let table = TransitionTable::from_ioa_source(ITEM_IOA); + let present_set = declared_vector_set_signature(&table.vectors); + store.persist_spec_declaration("default", "Item", &fingerprint); + + let state = state_with_item(&store, true, seed); + let embedding = + serde_json::to_string(&[1.0f32, 0.0, 0.0, 0.0]).expect("serialize deterministic embedding"); + let persistence_id = "default:Item:item-restart"; + let events = [ + EntityEvent { + action: "Created".to_string(), + from_status: String::new(), + to_status: "New".to_string(), + timestamp: sim_now(), + params: serde_json::json!({}), + idempotency_key: None, + }, + EntityEvent { + action: "Create".to_string(), + from_status: "New".to_string(), + to_status: "Ready".to_string(), + timestamp: sim_now(), + params: serde_json::json!({ + "Embedding": embedding, + "EmbeddingModel": "m1", + }), + idempotency_key: None, + }, + ]; + let envelopes = events + .iter() + .enumerate() + .map(|(index, event)| PersistenceEnvelope { + sequence_nr: (index + 1) as u64, + event_type: event.action.clone(), + payload: serde_json::to_value(event).expect("serialize entity event"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: event.timestamp, + actor_id: persistence_id.to_string(), + }, + }) + .collect::>(); + store + .append_with_index_rows( + persistence_id, + 0, + &envelopes, + &[], + &[EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0, 0.0, 0.0], + }], + true, + Some(&fingerprint), + ) + .await + .expect("seed retained Item journal and vector row"); + state.populate_vector_index_from_snapshots(&tenant).await; + assert_eq!( + store + .vector_index_backfilled_types("default") + .await + .expect("read initial completion"), + vec![("Item".to_string(), present_set.clone())] + ); + + store.persist_spec_declaration("default", "Item", "absent:v1"); + state + .registry + .write() + .expect("registry lock") + .register_tenant( + "default", + parse_csdl(ITEM_CSDL).expect("parse deletion CSDL"), + ITEM_CSDL.to_string(), + &[], + ); + store.fail_next_reads("default:Item:item-restart", 1); + state.populate_vector_index_from_snapshots(&tenant).await; + assert!( + store + .vector_index_backfilled_types("default") + .await + .expect("read interrupted deletion completion") + .is_empty(), + "seed {seed}: a crashed purge must not publish completion" + ); + + drop(state); + let restarted = state_with_item(&store, false, seed + 1); + restarted + .populate_vector_index_from_snapshots(&tenant) + .await; + assert_eq!( + store + .vector_index_backfilled_types("default") + .await + .expect("read resumed deletion completion"), + vec![("Item".to_string(), "v2|".to_string())], + "seed {seed}: rebuilt registry revision one must resume the durable tombstone" + ); + assert!( + store + .vector_candidates("default", "Item", "embed", "m1", 10) + .await + .expect("read purged candidates") + .is_empty(), + "seed {seed}: absent reconciliation must purge retained rows" + ); + + store.persist_spec_declaration("default", "Item", &fingerprint); + restarted + .registry + .write() + .expect("registry lock") + .register_tenant( + "default", + parse_csdl(ITEM_CSDL).expect("parse re-add CSDL"), + ITEM_CSDL.to_string(), + &[("Item", ITEM_IOA)], + ); + restarted + .populate_vector_index_from_snapshots(&tenant) + .await; + assert_eq!( + store + .vector_index_backfilled_types("default") + .await + .expect("read re-add completion"), + vec![("Item".to_string(), present_set)], + "seed {seed}: identical re-add must claim a newer durable declaration" + ); + assert_eq!( + store + .vector_candidates("default", "Item", "embed", "m1", 10) + .await + .expect("read rebuilt candidates") + .len(), + 1, + "seed {seed}: re-add must rebuild the retained journal stream" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn malformed_journal_event_blocks_vector_completion_watermark() { + let seed = 217; + let (_guard, _clock, _id) = install_deterministic_context(seed); + let store = SimEventStore::no_faults(seed); + let state = state_with_item(&store, true, seed); + let fingerprint = temper_store_turso::spec_content_hash(ITEM_IOA); + store.persist_spec_declaration("default", "Item", &fingerprint); + store + .append_with_index_rows( + "default:Item:item-malformed", + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "Create".to_string(), + payload: serde_json::json!({"incompatible": true}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: "default:Item:item-malformed".to_string(), + }, + }], + &[], + &[], + false, + Some(&fingerprint), + ) + .await + .expect("seed malformed durable envelope"); + + state + .populate_vector_index_from_snapshots(&TenantId::default()) + .await; + assert!( + store + .vector_index_backfilled_types("default") + .await + .expect("read completion claims") + .is_empty(), + "strict replay must not watermark a type after skipping an incompatible event" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn truncated_journal_fault_blocks_vector_completion_watermark() { + let seed = 218; + let (_guard, _clock, _id) = install_deterministic_context(seed); + let store = SimEventStore::new( + seed, + SimFaultConfig { + write_failure_prob: 0.0, + concurrency_violation_prob: 0.0, + read_truncation_prob: 1.0, + snapshot_failure_prob: 0.0, + }, + ); + let state = state_with_item(&store, true, seed); + let fingerprint = temper_store_turso::spec_content_hash(ITEM_IOA); + store.persist_spec_declaration("default", "Item", &fingerprint); + let persistence_id = "default:Item:item-truncated"; + let events = [ + EntityEvent { + action: "Created".to_string(), + from_status: String::new(), + to_status: "New".to_string(), + timestamp: sim_now(), + params: serde_json::json!({}), + idempotency_key: None, + }, + EntityEvent { + action: "Create".to_string(), + from_status: "New".to_string(), + to_status: "Ready".to_string(), + timestamp: sim_now(), + params: serde_json::json!({ + "Embedding": "[1.0,0.0,0.0,0.0]", + "EmbeddingModel": "m1", + }), + idempotency_key: None, + }, + ]; + let envelopes = events + .iter() + .map(|event| PersistenceEnvelope { + sequence_nr: 0, + event_type: event.action.clone(), + payload: serde_json::to_value(event).expect("serialize entity event"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: event.timestamp, + actor_id: persistence_id.to_string(), + }, + }) + .collect::>(); + store + .append_with_index_rows( + persistence_id, + 0, + &envelopes, + &[], + &[], + false, + Some(&fingerprint), + ) + .await + .expect("seed two-event journal"); + + state + .populate_vector_index_from_snapshots(&TenantId::default()) + .await; + assert!( + store + .vector_index_backfilled_types("default") + .await + .expect("read completion claims") + .is_empty(), + "a modeled truncated prefix must surface as failure, never as complete replay" + ); +} diff --git a/crates/temper-server/tests/e2e_gepa_loop.rs b/crates/temper-server/tests/e2e_gepa_loop.rs index d7f97ed28..5ddf1084d 100644 --- a/crates/temper-server/tests/e2e_gepa_loop.rs +++ b/crates/temper-server/tests/e2e_gepa_loop.rs @@ -927,30 +927,9 @@ hint = "Reassign the issue to a different implementer." parsed.err() ); - // Hot-deploy: re-register the tenant with the mutated Issue spec (merge mode). - { - let mut registry = harness.platform_state.registry.write().unwrap(); // ci-ok: infallible lock - let tenant_id = temper_runtime::tenant::TenantId::new(TENANT); - // Get existing CSDL for merge. - let existing_csdl = registry - .get_tenant(&tenant_id) - .expect("tenant should exist") - .csdl - .as_ref() - .clone(); - let csdl_xml = temper_spec::csdl::emit_csdl_xml(&existing_csdl); - registry - .try_register_tenant_with_reactions_and_constraints( - tenant_id, - existing_csdl, - csdl_xml, - &[("Issue", &mutated_issue_spec)], - Vec::new(), - None, - true, // merge mode — only update Issue, preserve others - ) - .expect("hot-deploy should succeed"); - } + // Hot-deploy through the harness primitive that advances durable + // declaration authority before publishing the replacement table. + harness.register_inline_spec(TENANT, "Issue", &mutated_issue_spec); // Now Reassign should work on an Issue that has an assignee set. // Create a fresh Issue (starts in Backlog), then Assign to set assignee_set=true. @@ -1181,28 +1160,7 @@ params = ["NewAssigneeId"] hint = "Reassign the issue to a different implementer." "#; - { - let mut registry = harness.platform_state.registry.write().unwrap(); // ci-ok: infallible lock - let tenant_id = temper_runtime::tenant::TenantId::new(TENANT); - let existing_csdl = registry - .get_tenant(&tenant_id) - .expect("tenant should exist") - .csdl - .as_ref() - .clone(); - let csdl_xml = temper_spec::csdl::emit_csdl_xml(&existing_csdl); - registry - .try_register_tenant_with_reactions_and_constraints( - tenant_id, - existing_csdl, - csdl_xml, - &[("Issue", &mutated_issue_spec)], - Vec::new(), - None, - true, // merge mode - ) - .expect("hot-deploy should succeed"); - } + harness.register_inline_spec(TENANT, "Issue", &mutated_issue_spec); // Complete the deployment. let r = harness diff --git a/crates/temper-server/tests/fixtures/arn216/full_v1/item.ioa.toml b/crates/temper-server/tests/fixtures/arn216/full_v1/item.ioa.toml new file mode 100644 index 000000000..b8d805da4 --- /dev/null +++ b/crates/temper-server/tests/fixtures/arn216/full_v1/item.ioa.toml @@ -0,0 +1,16 @@ +[automaton] +name = "Item" +states = ["New", "Ready"] +initial = "New" + +[[state]] +name = "Title" +type = "string" +initial = "" + +[[action]] +name = "Create" +kind = "input" +from = ["New"] +to = "Ready" +params = ["Title"] diff --git a/crates/temper-server/tests/fixtures/arn216/full_v1/model.csdl.xml b/crates/temper-server/tests/fixtures/arn216/full_v1/model.csdl.xml new file mode 100644 index 000000000..840d37050 --- /dev/null +++ b/crates/temper-server/tests/fixtures/arn216/full_v1/model.csdl.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/crates/temper-server/tests/fixtures/arn216/full_v1/note.ioa.toml b/crates/temper-server/tests/fixtures/arn216/full_v1/note.ioa.toml new file mode 100644 index 000000000..6bdb4bae7 --- /dev/null +++ b/crates/temper-server/tests/fixtures/arn216/full_v1/note.ioa.toml @@ -0,0 +1,16 @@ +[automaton] +name = "Note" +states = ["Draft", "Published"] +initial = "Draft" + +[[state]] +name = "Body" +type = "string" +initial = "" + +[[action]] +name = "Publish" +kind = "input" +from = ["Draft"] +to = "Published" +params = ["Body"] diff --git a/crates/temper-server/tests/fixtures/arn216/full_v2/item.ioa.toml b/crates/temper-server/tests/fixtures/arn216/full_v2/item.ioa.toml new file mode 100644 index 000000000..b8d805da4 --- /dev/null +++ b/crates/temper-server/tests/fixtures/arn216/full_v2/item.ioa.toml @@ -0,0 +1,16 @@ +[automaton] +name = "Item" +states = ["New", "Ready"] +initial = "New" + +[[state]] +name = "Title" +type = "string" +initial = "" + +[[action]] +name = "Create" +kind = "input" +from = ["New"] +to = "Ready" +params = ["Title"] diff --git a/crates/temper-server/tests/fixtures/arn216/full_v2/model.csdl.xml b/crates/temper-server/tests/fixtures/arn216/full_v2/model.csdl.xml new file mode 100644 index 000000000..aa0f41ef8 --- /dev/null +++ b/crates/temper-server/tests/fixtures/arn216/full_v2/model.csdl.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/crates/temper-server/tests/fixtures/arn216/full_v2/note.ioa.toml b/crates/temper-server/tests/fixtures/arn216/full_v2/note.ioa.toml new file mode 100644 index 000000000..2f09eeca8 --- /dev/null +++ b/crates/temper-server/tests/fixtures/arn216/full_v2/note.ioa.toml @@ -0,0 +1,28 @@ +[automaton] +name = "Note" +states = ["Draft", "Reviewed", "Published"] +initial = "Draft" + +[[state]] +name = "Body" +type = "string" +initial = "" + +[[state]] +name = "Revision" +type = "string" +initial = "v2" + +[[action]] +name = "Review" +kind = "input" +from = ["Draft"] +to = "Reviewed" +params = ["Body"] + +[[action]] +name = "Publish" +kind = "input" +from = ["Reviewed"] +to = "Published" +params = [] diff --git a/crates/temper-server/tests/fixtures/arn216/item_only/item.ioa.toml b/crates/temper-server/tests/fixtures/arn216/item_only/item.ioa.toml new file mode 100644 index 000000000..b8d805da4 --- /dev/null +++ b/crates/temper-server/tests/fixtures/arn216/item_only/item.ioa.toml @@ -0,0 +1,16 @@ +[automaton] +name = "Item" +states = ["New", "Ready"] +initial = "New" + +[[state]] +name = "Title" +type = "string" +initial = "" + +[[action]] +name = "Create" +kind = "input" +from = ["New"] +to = "Ready" +params = ["Title"] diff --git a/crates/temper-server/tests/fixtures/arn216/item_only/model.csdl.xml b/crates/temper-server/tests/fixtures/arn216/item_only/model.csdl.xml new file mode 100644 index 000000000..5c89b8daf --- /dev/null +++ b/crates/temper-server/tests/fixtures/arn216/item_only/model.csdl.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/crates/temper-server/tests/gepa_manual_verification.rs b/crates/temper-server/tests/gepa_manual_verification.rs index 426c1f038..44e9f7f0d 100644 --- a/crates/temper-server/tests/gepa_manual_verification.rs +++ b/crates/temper-server/tests/gepa_manual_verification.rs @@ -683,31 +683,10 @@ hint = "Reassign the issue to a different implementer." Err(e) => println!(" Mutated spec: PARSE FAILED — {e}"), } - // Hot-deploy via registry merge - { - let mut registry = harness.platform_state.registry.write().unwrap(); // ci-ok: infallible lock - let tenant_id = temper_runtime::tenant::TenantId::new(TENANT); - let existing_csdl = registry - .get_tenant(&tenant_id) - .expect("tenant") - .csdl - .as_ref() - .clone(); - let csdl_xml = temper_spec::csdl::emit_csdl_xml(&existing_csdl); - let deploy_result = registry.try_register_tenant_with_reactions_and_constraints( - tenant_id, - existing_csdl, - csdl_xml, - &[("Issue", &mutated_spec)], - Vec::new(), - None, - true, - ); - match &deploy_result { - Ok(()) => println!(" Hot-deploy: SUCCESS"), - Err(e) => println!(" Hot-deploy: FAILED — {e}"), - } - } + // Hot-deploy through the same test primitive used by the behavioral + // suite so durable declaration authority precedes registry publication. + harness.register_inline_spec(TENANT, "Issue", &mutated_spec); + println!(" Hot-deploy: SUCCESS"); // Assign first (to satisfy guard is_true assignee_set) let r = harness diff --git a/crates/temper-server/tests/nearest_odata.rs b/crates/temper-server/tests/nearest_odata.rs index 240569bd6..03d42f2e0 100644 --- a/crates/temper-server/tests/nearest_odata.rs +++ b/crates/temper-server/tests/nearest_odata.rs @@ -4,9 +4,13 @@ //! and assert the OData list shape, ranking order, per-row `@temper.score`, and //! self-exclusion. +use std::collections::BTreeMap; + use axum::body::Body; use axum::http::{Request, StatusCode}; use temper_runtime::ActorSystem; +use temper_runtime::persistence::EventStore; +use temper_runtime::scheduler::sim_uuid; use temper_runtime::tenant::TenantId; use temper_server::build_router; use temper_server::registry::SpecRegistry; @@ -14,6 +18,7 @@ use temper_server::request_context::AgentContext; use temper_server::{ServerState, StorageStack}; use temper_spec::csdl::parse_csdl; use temper_store_sim::SimEventStore; +use temper_store_turso::TursoEventStore; use tower::ServiceExt; const VEC_ITEM_IOA: &str = r#" @@ -80,7 +85,7 @@ const CSDL_XML: &str = r#" "#; -fn build_state() -> ServerState { +fn build_state_with_store(system_name: &str) -> (ServerState, SimEventStore) { let mut registry = SpecRegistry::new(); let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); registry.register_tenant( @@ -89,10 +94,15 @@ fn build_state() -> ServerState { CSDL_XML.to_string(), &[("VecItem", VEC_ITEM_IOA)], ); - let system = ActorSystem::new("nearest-odata"); + let system = ActorSystem::new(system_name); + let store = SimEventStore::no_faults(7); let mut state = ServerState::from_registry(system, registry); - state.set_storage_stack(StorageStack::from_sim(SimEventStore::no_faults(7), None)); - state + state.set_storage_stack(StorageStack::from_sim(store.clone(), None)); + (state, store) +} + +fn build_state() -> ServerState { + build_state_with_store("nearest-odata").0 } async fn create_item( @@ -303,6 +313,71 @@ async fn nearest_applies_equality_filter_before_top_k() { ); } +#[tokio::test] +async fn vector_backfill_retries_when_watermark_persistence_fails() { + let (state, store) = build_state_with_store("vector-watermark-failure"); + let tenant = TenantId::from("default"); + create_item(&state, &tenant, "item-a", &[1.0, 0.0, 0.0, 0.0], "m1").await; + + store.fail_next_vector_watermarks(tenant.as_str(), "VecItem", 1); + state.populate_vector_index_from_snapshots(&tenant).await; + assert!( + store + .vector_index_backfilled_types(tenant.as_str()) + .await + .expect("read vector watermark after injected failure") + .is_empty(), + "a failed durable watermark write must never advertise completion" + ); + + state.populate_vector_index_from_snapshots(&tenant).await; + assert_eq!( + store + .vector_index_backfilled_types(tenant.as_str()) + .await + .expect("read vector watermark after retry"), + vec![( + "VecItem".to_string(), + "v2|embed:Embedding:EmbeddingModel:4:cosine".to_string(), + )], + "the next run must repeat reconciliation and persist the convergence claim" + ); +} + +#[tokio::test] +async fn compatibility_storage_constructor_bootstraps_fresh_store_authority() { + let db_path = std::env::temp_dir().join(format!( + "temper-vector-compat-constructor-{}.db", + sim_uuid() + )); + let db_url = format!("file:{}", db_path.display()); + let store = TursoEventStore::new(&db_url, None) + .await + .expect("create local Turso store"); + let state = ServerState::with_storage_stack( + ActorSystem::new("vector-compat-constructor"), + parse_csdl(CSDL_XML).expect("CSDL parse"), + CSDL_XML.to_string(), + BTreeMap::from([("VecItem".to_string(), VEC_ITEM_IOA.to_string())]), + StorageStack::from_turso(store.clone()), + ) + .expect("construct compatibility server state"); + let tenant = TenantId::default(); + create_item(&state, &tenant, "item-a", &[1.0, 0.0, 0.0, 0.0], "m1").await; + state.populate_vector_index_from_snapshots(&tenant).await; + + assert_eq!( + store + .vector_index_backfilled_types("default") + .await + .expect("read vector completion"), + vec![( + "VecItem".to_string(), + "v2|embed:Embedding:EmbeddingModel:4:cosine".to_string(), + )] + ); +} + #[tokio::test] async fn nearest_authorizes_reference_and_walk_rows() { let state = build_state(); diff --git a/crates/temper-server/tests/storage_stack.rs b/crates/temper-server/tests/storage_stack.rs index e49d338b7..f0d590dbf 100644 --- a/crates/temper-server/tests/storage_stack.rs +++ b/crates/temper-server/tests/storage_stack.rs @@ -196,6 +196,9 @@ async fn boxed_event_store_delegates_through_object_safe_adapter() { persistence_id: "default:Ticket:t-1".to_string(), expected_sequence: 0, events: events.clone(), + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, }]) .await .expect("append batch through dyn adapter"), diff --git a/crates/temper-store-postgres/migrations/0013_monotonic_vector_reconciliation.sql b/crates/temper-store-postgres/migrations/0013_monotonic_vector_reconciliation.sql new file mode 100644 index 000000000..3fee4a494 --- /dev/null +++ b/crates/temper-store-postgres/migrations/0013_monotonic_vector_reconciliation.sql @@ -0,0 +1,308 @@ +-- ADR-0181: retain one journal-sequence fence per vector-indexed entity. +-- +-- The row survives when the entity's vector set is empty. Backfill transactions +-- compare their observed journal sequence against this fence before replacing any +-- rows, so a rebuild that loaded N cannot overwrite a live append committed at N+1. +CREATE TABLE IF NOT EXISTS entity_vector_index_version ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id TEXT NOT NULL, + reconciliation_generation BIGINT NOT NULL DEFAULT 0, + sequence_nr BIGINT NOT NULL, + PRIMARY KEY (tenant, entity_type, entity_id) +); + +ALTER TABLE entity_vector_index_version + ADD COLUMN IF NOT EXISTS reconciliation_generation BIGINT NOT NULL DEFAULT 0; + +-- Durable ordering for overlapping declaration-set reconciliations. Every entity +-- replacement and final watermark must carry the current generation. +CREATE TABLE IF NOT EXISTS entity_vector_reconciliation_generation ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + generation BIGINT NOT NULL, + declaration_revision BIGINT NOT NULL DEFAULT 0, + declaration_fingerprint TEXT NOT NULL DEFAULT '', + vector_set TEXT NOT NULL, + PRIMARY KEY (tenant, entity_type) +); + +ALTER TABLE entity_vector_reconciliation_generation + ADD COLUMN IF NOT EXISTS declaration_revision BIGINT NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS declaration_fingerprint TEXT NOT NULL DEFAULT ''; + +-- Durable declaration authority is separate from reconciliation state so ordinary +-- non-vector entity types do not become vector-repair work merely because their +-- spec exists. Tombstones deliberately survive hard deletion and preserve the +-- per-type revision across delete/re-add cycles. +CREATE TABLE IF NOT EXISTS spec_declaration_authority ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + revision BIGINT NOT NULL, + ioa_source TEXT NOT NULL DEFAULT '', + declaration_fingerprint TEXT NOT NULL DEFAULT '', + present BOOLEAN NOT NULL, + PRIMARY KEY (tenant, entity_type) +); + +ALTER TABLE spec_declaration_authority + ADD COLUMN IF NOT EXISTS declaration_fingerprint TEXT NOT NULL DEFAULT ''; + +-- All reconciliation metadata is tenant-owned state. Keep these statements +-- idempotent because local databases may have created the tables while this +-- migration was under development. +ALTER TABLE entity_vector_index_version ENABLE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON entity_vector_index_version; +CREATE POLICY tenant_isolation ON entity_vector_index_version + USING (tenant = current_setting('app.current_tenant', true)) + WITH CHECK (tenant = current_setting('app.current_tenant', true)); + +ALTER TABLE entity_vector_reconciliation_generation ENABLE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON entity_vector_reconciliation_generation; +CREATE POLICY tenant_isolation ON entity_vector_reconciliation_generation + USING (tenant = current_setting('app.current_tenant', true)) + WITH CHECK (tenant = current_setting('app.current_tenant', true)); + +ALTER TABLE spec_declaration_authority ENABLE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON spec_declaration_authority; +CREATE POLICY tenant_isolation ON spec_declaration_authority + USING (tenant = current_setting('app.current_tenant', true)) + WITH CHECK (tenant = current_setting('app.current_tenant', true)); + +-- Preserve the strongest sequence already present when upgrading an existing +-- index. Rows written by the legacy backfill carry sequence 0 and are deliberately +-- rebuilt once through the revisioned watermark protocol. +INSERT INTO entity_vector_index_version + (tenant, entity_type, entity_id, reconciliation_generation, sequence_nr) +SELECT tenant, entity_type, entity_id, 0, MAX(sequence_nr) +FROM entity_vector_index +GROUP BY tenant, entity_type, entity_id +ON CONFLICT (tenant, entity_type, entity_id) +DO UPDATE SET + reconciliation_generation = GREATEST( + entity_vector_index_version.reconciliation_generation, + EXCLUDED.reconciliation_generation + ), + sequence_nr = CASE + WHEN entity_vector_index_version.reconciliation_generation + = EXCLUDED.reconciliation_generation + THEN GREATEST(entity_vector_index_version.sequence_nr, EXCLUDED.sequence_nr) + ELSE entity_vector_index_version.sequence_nr + END; + +-- Seed current specs and tombstones for legacy vector state. A type can have no +-- current spec yet still require one final empty reconciliation to purge retained +-- candidates or an old completion watermark. +INSERT INTO spec_declaration_authority + (tenant, entity_type, revision, ioa_source, declaration_fingerprint, present) +SELECT tenant, entity_type, GREATEST(version::BIGINT, 1), ioa_source, content_hash, true +FROM specs +WHERE committed = true +ON CONFLICT (tenant, entity_type) DO NOTHING; + +INSERT INTO spec_declaration_authority + (tenant, entity_type, revision, ioa_source, declaration_fingerprint, present) +SELECT known.tenant, known.entity_type, 1, '', 'absent:v1', false +FROM ( + SELECT tenant, entity_type FROM entity_vector_index + UNION + SELECT tenant, entity_type FROM entity_vector_index_version + UNION + SELECT tenant, entity_type FROM entity_vector_reconciliation_generation + UNION + SELECT tenant, entity_type FROM vector_index_backfill_watermark +) AS known +WHERE NOT EXISTS ( + SELECT 1 + FROM specs + WHERE specs.tenant = known.tenant + AND specs.entity_type = known.entity_type + AND specs.committed = true +) +ON CONFLICT (tenant, entity_type) DO NOTHING; + +-- Upgrade authority rows created by an earlier development version of this +-- migration. New catalog rows carry their exact content hash; a blank value is +-- retained only for legacy catalogs whose hash was never persisted, allowing +-- the runtime to derive SHA-256 from ioa_source without encoding it as source. +UPDATE spec_declaration_authority AS authority +SET declaration_fingerprint = specs.content_hash +FROM specs +WHERE authority.tenant = specs.tenant + AND authority.entity_type = specs.entity_type + AND authority.present + AND authority.declaration_fingerprint = '' + AND specs.committed = true + AND specs.content_hash <> ''; + +UPDATE spec_declaration_authority +SET declaration_fingerprint = 'absent:v1' +WHERE NOT present + AND declaration_fingerprint = ''; + +-- Committed spec mutation is the declaration-order publication point. Staged +-- `committed = false` rows remain invisible to live writers and vector work; +-- the false-to-true transition advances durable authority and immediately +-- fences an existing vector rebuild before registry publication. +CREATE OR REPLACE FUNCTION advance_spec_declaration_authority() +RETURNS TRIGGER AS $$ +DECLARE + authority_tenant TEXT; + authority_entity_type TEXT; + authority_source TEXT; + authority_fingerprint TEXT; + authority_present BOOLEAN; + next_revision BIGINT; +BEGIN + IF TG_OP = 'DELETE' THEN + authority_tenant := OLD.tenant; + authority_entity_type := OLD.entity_type; + authority_source := ''; + authority_fingerprint := 'absent:v1'; + authority_present := false; + ELSE + authority_tenant := NEW.tenant; + authority_entity_type := NEW.entity_type; + authority_source := NEW.ioa_source; + authority_fingerprint := NEW.content_hash; + authority_present := true; + END IF; + + -- Serialize catalog mutation, compatibility bootstrap, and tombstoning for + -- one tenant/type even before an authority row exists to take a row lock. + PERFORM pg_advisory_xact_lock( + hashtextextended(authority_tenant || ':' || authority_entity_type, 0) + ); + + INSERT INTO spec_declaration_authority + (tenant, entity_type, revision, ioa_source, declaration_fingerprint, present) + VALUES ( + authority_tenant, + authority_entity_type, + 1, + authority_source, + authority_fingerprint, + authority_present + ) + ON CONFLICT (tenant, entity_type) DO UPDATE SET + revision = spec_declaration_authority.revision + 1, + ioa_source = EXCLUDED.ioa_source, + declaration_fingerprint = EXCLUDED.declaration_fingerprint, + present = EXCLUDED.present + RETURNING revision INTO next_revision; + + UPDATE entity_vector_reconciliation_generation + SET generation = generation + 1, + declaration_revision = next_revision, + declaration_fingerprint = '', + vector_set = '' + WHERE tenant = authority_tenant + AND entity_type = authority_entity_type; + + -- A completion claim is invalid as soon as declaration authority changes, + -- even when no reconciliation-generation row has been created yet. + DELETE FROM vector_index_backfill_watermark + WHERE tenant = authority_tenant + AND entity_type = authority_entity_type; + + IF TG_OP = 'DELETE' THEN + RETURN OLD; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS specs_declaration_authority_insert ON specs; +CREATE TRIGGER specs_declaration_authority_insert +AFTER INSERT ON specs +FOR EACH ROW +WHEN (NEW.committed IS TRUE) +EXECUTE FUNCTION advance_spec_declaration_authority(); + +DROP TRIGGER IF EXISTS specs_declaration_authority_update ON specs; +CREATE TRIGGER specs_declaration_authority_update +AFTER UPDATE OF ioa_source, content_hash, committed ON specs +FOR EACH ROW +WHEN ( + NEW.committed IS TRUE + AND ( + OLD.committed IS DISTINCT FROM TRUE + OR OLD.ioa_source IS DISTINCT FROM NEW.ioa_source + OR OLD.content_hash IS DISTINCT FROM NEW.content_hash + ) +) +EXECUTE FUNCTION advance_spec_declaration_authority(); + +DROP TRIGGER IF EXISTS specs_declaration_authority_delete ON specs; +CREATE TRIGGER specs_declaration_authority_delete +AFTER DELETE ON specs +FOR EACH ROW +WHEN (OLD.committed IS TRUE) +EXECUTE FUNCTION advance_spec_declaration_authority(); + +-- Full replacement must retain declaration absence even when compatibility +-- constructors bootstrapped authority without ever creating a specs row. +-- Callers with only a PgPool invoke: +-- SELECT tombstone_spec_declaration_authority($1, $2) +CREATE OR REPLACE FUNCTION tombstone_spec_declaration_authority( + target_tenant TEXT, + target_entity_type TEXT +) +RETURNS VOID AS $$ +DECLARE + deleted_catalog_committed BOOLEAN; + next_revision BIGINT; +BEGIN + PERFORM pg_advisory_xact_lock( + hashtextextended(target_tenant || ':' || target_entity_type, 0) + ); + + DELETE FROM specs + WHERE tenant = target_tenant + AND entity_type = target_entity_type + RETURNING committed INTO deleted_catalog_committed; + + -- Only deletion of a committed row fires the authority trigger. An + -- uncommitted staging row can hide the committed declaration represented + -- by authority, so its deletion must fall through to the explicit + -- tombstone below. + IF deleted_catalog_committed IS TRUE THEN + RETURN; + END IF; + + INSERT INTO spec_declaration_authority + (tenant, entity_type, revision, ioa_source, declaration_fingerprint, present) + VALUES ( + target_tenant, + target_entity_type, + 1, + '', + 'absent:v1', + false + ) + ON CONFLICT (tenant, entity_type) DO UPDATE SET + revision = spec_declaration_authority.revision + 1, + ioa_source = '', + declaration_fingerprint = 'absent:v1', + present = false + WHERE spec_declaration_authority.present + RETURNING revision INTO next_revision; + + -- Repeating an already-persisted tombstone is an idempotent no-op. + IF next_revision IS NULL THEN + RETURN; + END IF; + + UPDATE entity_vector_reconciliation_generation + SET generation = generation + 1, + declaration_revision = next_revision, + declaration_fingerprint = '', + vector_set = '' + WHERE tenant = target_tenant + AND entity_type = target_entity_type; + + DELETE FROM vector_index_backfill_watermark + WHERE tenant = target_tenant + AND entity_type = target_entity_type; +END; +$$ LANGUAGE plpgsql; diff --git a/crates/temper-store-postgres/migrations/0014_versioned_spec_staging.sql b/crates/temper-store-postgres/migrations/0014_versioned_spec_staging.sql new file mode 100644 index 000000000..805531b15 --- /dev/null +++ b/crates/temper-store-postgres/migrations/0014_versioned_spec_staging.sql @@ -0,0 +1,37 @@ +-- Preserve the last committed catalog while replacement bytes are verified. +-- +-- A staging row must never overwrite the only restorable committed spec. The +-- verifier promotes one exact IOA+CSDL pair into `specs` atomically. +CREATE TABLE IF NOT EXISTS staged_specs ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + ioa_source TEXT NOT NULL, + csdl_xml TEXT, + content_hash TEXT NOT NULL, + version INT NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (tenant, entity_type) +); + +-- Preserve interrupted staging rows created by the previous single-row +-- protocol. Those rows never carried committed authority. +INSERT INTO staged_specs + (tenant, entity_type, ioa_source, csdl_xml, content_hash, version, created_at, updated_at) +SELECT tenant, entity_type, ioa_source, csdl_xml, content_hash, version, created_at, updated_at +FROM specs +WHERE committed = false +ON CONFLICT (tenant, entity_type) DO UPDATE SET + ioa_source = EXCLUDED.ioa_source, + csdl_xml = EXCLUDED.csdl_xml, + content_hash = EXCLUDED.content_hash, + version = EXCLUDED.version, + updated_at = EXCLUDED.updated_at; + +DELETE FROM specs WHERE committed = false; + +ALTER TABLE staged_specs ENABLE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS tenant_isolation ON staged_specs; +CREATE POLICY tenant_isolation ON staged_specs + USING (tenant = current_setting('app.current_tenant', true)) + WITH CHECK (tenant = current_setting('app.current_tenant', true)); diff --git a/crates/temper-store-postgres/src/data_only_create.rs b/crates/temper-store-postgres/src/data_only_create.rs index c1b3e2c6b..039a004ae 100644 --- a/crates/temper-store-postgres/src/data_only_create.rs +++ b/crates/temper-store-postgres/src/data_only_create.rs @@ -47,6 +47,7 @@ impl PostgresEventStore { fields, &state, event, + None, ) .await } @@ -65,6 +66,7 @@ impl PostgresEventStore { fields: &serde_json::Value, state: &serde_json::Value, event: &PersistenceEnvelope, + spec_declaration_fingerprint: Option<&str>, ) -> Result { assert_eq!( event.sequence_nr, 1, @@ -120,6 +122,10 @@ impl PostgresEventStore { } }; + if let Some(fingerprint) = spec_declaration_fingerprint { + Self::validate_live_spec_declaration(&mut tx, tenant, entity_type, fingerprint).await?; + } + let metadata_json = serde_json::to_value(&event.metadata) .map_err(|e| PersistenceError::Serialization(e.to_string()))?; if let Err(e) = crate::dbm::postgres_query!( diff --git a/crates/temper-store-postgres/src/lib.rs b/crates/temper-store-postgres/src/lib.rs index af25937ce..62f1939d0 100644 --- a/crates/temper-store-postgres/src/lib.rs +++ b/crates/temper-store-postgres/src/lib.rs @@ -22,6 +22,7 @@ pub mod schema; mod schema_event_history; mod segments; mod selected_catalog; +mod spec_catalog; pub mod store; pub use metrics::init_metrics; diff --git a/crates/temper-store-postgres/src/migration.rs b/crates/temper-store-postgres/src/migration.rs index 5a40c7493..2f2131442 100644 --- a/crates/temper-store-postgres/src/migration.rs +++ b/crates/temper-store-postgres/src/migration.rs @@ -37,6 +37,12 @@ mod tests { include_str!("../migrations/0006_segmented_event_history.sql"), include_str!("../migrations/0007_installed_app_follow_policy.sql"), include_str!("../migrations/0008_ots_trajectory_outbox_status.sql"), + include_str!("../migrations/0009_entity_key_index.sql"), + include_str!("../migrations/0010_key_index_backfill_watermark.sql"), + include_str!("../migrations/0011_key_index_watermark_key_set.sql"), + include_str!("../migrations/0012_entity_vector_index.sql"), + include_str!("../migrations/0013_monotonic_vector_reconciliation.sql"), + include_str!("../migrations/0014_versioned_spec_staging.sql"), ] .join("\n") .to_lowercase(); @@ -53,6 +59,12 @@ mod tests { "event_segments", "snapshot_history", "ots_trajectories", + "entity_key_index", + "entity_vector_index", + "entity_vector_index_version", + "entity_vector_reconciliation_generation", + "spec_declaration_authority", + "staged_specs", ] { assert!( migration.contains(&format!("create table if not exists {table}")), @@ -103,6 +115,114 @@ mod tests { ); } + #[test] + fn migration_thirteen_is_tenant_scoped_and_always_withdraws_stale_watermarks() { + let migration = + include_str!("../migrations/0013_monotonic_vector_reconciliation.sql").to_lowercase(); + for table in [ + "entity_vector_index_version", + "entity_vector_reconciliation_generation", + "spec_declaration_authority", + ] { + assert!( + migration.contains(&format!("alter table {table} enable row level security")), + "migration 0013 must enable RLS for {table}" + ); + assert!( + migration.contains(&format!( + "drop policy if exists tenant_isolation on {table}" + )), + "migration 0013 tenant policy must be idempotent for {table}" + ); + assert!( + migration.contains(&format!("create policy tenant_isolation on {table}")), + "migration 0013 must create tenant isolation for {table}" + ); + } + + let authority_trigger = migration + .split("create or replace function advance_spec_declaration_authority()") + .nth(1) + .expect("migration 0013 declaration authority trigger") + .split("drop trigger if exists specs_declaration_authority_insert") + .next() + .expect("migration 0013 declaration authority function body"); + assert!( + authority_trigger.contains("delete from vector_index_backfill_watermark"), + "every durable declaration change must withdraw the completion watermark" + ); + assert!( + !authority_trigger.contains("if found then"), + "watermark withdrawal must not depend on an existing generation row" + ); + assert!( + migration.contains( + "add column if not exists declaration_fingerprint text not null default ''" + ), + "declaration authority must retain the exact persisted fingerprint" + ); + assert!( + migration.contains( + "select tenant, entity_type, greatest(version::bigint, 1), ioa_source, content_hash, true" + ), + "legacy authority seeding must prefer the specs content hash" + ); + assert!( + authority_trigger.contains("authority_fingerprint := new.content_hash"), + "spec triggers must copy the catalog fingerprint into declaration authority" + ); + assert!( + authority_trigger.contains("authority_fingerprint := 'absent:v1'"), + "spec deletion must leave an explicit declaration tombstone fingerprint" + ); + let update_trigger = migration + .split("create trigger specs_declaration_authority_update") + .nth(1) + .expect("migration 0013 declaration authority update trigger") + .split("execute function advance_spec_declaration_authority()") + .next() + .expect("migration 0013 declaration authority update predicate"); + assert!( + update_trigger.contains("after update of ioa_source, content_hash, committed on specs"), + "content-hash-only catalog updates and staged commits must advance authority" + ); + assert!( + update_trigger.contains("new.committed is true") + && update_trigger.contains("old.committed is distinct from true"), + "only committed declarations, including false-to-true publication, may advance authority" + ); + assert!( + migration.contains("when (new.committed is true)") + && migration.contains("when (old.committed is true)"), + "staged inserts and deletes must remain invisible to declaration authority" + ); + assert!( + authority_trigger.contains("pg_advisory_xact_lock"), + "spec mutation must serialize with first-writer authority bootstrap" + ); + + let tombstone_function = migration + .split("create or replace function tombstone_spec_declaration_authority(") + .nth(1) + .expect("migration 0013 compatibility-authority tombstone function"); + assert!( + tombstone_function.contains("delete from specs"), + "the tombstone entry point must cover persisted catalogs" + ); + assert!( + tombstone_function.contains("on conflict (tenant, entity_type) do update set"), + "the tombstone entry point must cover first-writer authority without a catalog" + ); + assert!( + tombstone_function.contains("where spec_declaration_authority.present"), + "repeating an existing tombstone must be idempotent" + ); + assert!( + tombstone_function.contains("delete from vector_index_backfill_watermark"), + "tombstoning first-writer authority must withdraw completion" + ); + } + #[test] fn migration_sql_is_idempotent() { // Both schemas must use IF NOT EXISTS so repeated execution is safe. diff --git a/crates/temper-store-postgres/src/platform.rs b/crates/temper-store-postgres/src/platform.rs index 322e61fe9..0245fdeb6 100644 --- a/crates/temper-store-postgres/src/platform.rs +++ b/crates/temper-store-postgres/src/platform.rs @@ -1050,21 +1050,15 @@ impl PostgresEventStore { content_hash: &str, ) -> Result<(), PersistenceError> { crate::dbm::postgres_query!( - "INSERT INTO specs \ - (tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, verified, verification_status, updated_at) \ - VALUES ($1, $2, $3, $4, $5, false, 1, false, 'pending', now()) \ + "INSERT INTO staged_specs \ + (tenant, entity_type, ioa_source, csdl_xml, content_hash, version, updated_at) \ + VALUES ($1, $2, $3, $4, $5, 1, now()) \ ON CONFLICT (tenant, entity_type) DO UPDATE SET \ - ioa_source = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN EXCLUDED.ioa_source ELSE specs.ioa_source END, \ - csdl_xml = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN EXCLUDED.csdl_xml ELSE specs.csdl_xml END, \ - content_hash = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN EXCLUDED.content_hash ELSE specs.content_hash END, \ - committed = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN false ELSE specs.committed END, \ - version = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN specs.version + 1 ELSE specs.version END, \ - verified = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN false ELSE specs.verified END, \ - verification_status = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN 'pending' ELSE specs.verification_status END, \ - levels_passed = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN NULL ELSE specs.levels_passed END, \ - levels_total = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN NULL ELSE specs.levels_total END, \ - verification_result = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN NULL ELSE specs.verification_result END, \ - updated_at = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN now() ELSE specs.updated_at END", + ioa_source = EXCLUDED.ioa_source, \ + csdl_xml = EXCLUDED.csdl_xml, \ + content_hash = EXCLUDED.content_hash, \ + version = CASE WHEN staged_specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR staged_specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN staged_specs.version + 1 ELSE staged_specs.version END, \ + updated_at = CASE WHEN staged_specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR staged_specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN now() ELSE staged_specs.updated_at END", ) .bind(tenant) .bind(entity_type) @@ -1094,32 +1088,175 @@ impl PostgresEventStore { tenant: &str, entity_type: &str, ) -> Result<(), PersistenceError> { - crate::dbm::postgres_query!("DELETE FROM specs WHERE tenant = $1 AND entity_type = $2") + crate::dbm::postgres_query!( + "WITH staged AS ( \ + DELETE FROM staged_specs WHERE tenant = $1 AND entity_type = $2 \ + ) SELECT tombstone_spec_declaration_authority($1, $2)" + ) + .bind(tenant) + .bind(entity_type) + .execute(self.pool()) + .await + .map_err(storage_error)?; + Ok(()) + } + + pub async fn commit_specs(&self, tenant: &str) -> Result<(), PersistenceError> { + let mut tx = self.pool().begin().await.map_err(storage_error)?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended('spec-catalog:' || $1, 0))") + .bind(tenant) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + sqlx::query( + "WITH staged AS ( \ + DELETE FROM staged_specs WHERE tenant = $1 RETURNING * \ + ) \ + INSERT INTO specs \ + (tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, verified, verification_status, updated_at) \ + SELECT tenant, entity_type, ioa_source, csdl_xml, content_hash, true, version, false, 'pending', now() \ + FROM staged \ + ON CONFLICT (tenant, entity_type) DO UPDATE SET \ + ioa_source = EXCLUDED.ioa_source, csdl_xml = EXCLUDED.csdl_xml, \ + content_hash = EXCLUDED.content_hash, committed = true, \ + version = specs.version + 1, verified = false, verification_status = 'pending', \ + levels_passed = NULL, levels_total = NULL, verification_result = NULL, updated_at = now()" + ) + .bind(tenant) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + tx.commit().await.map_err(storage_error)?; + Ok(()) + } + + /// Atomically promote only staged specs matching one operation's exact bytes. + pub async fn commit_spec_batch( + &self, + tenant: &str, + expected: &[(&str, &str, &str)], + ) -> Result<(), PersistenceError> { + let mut expected = expected.to_vec(); + expected.sort_unstable_by(|left, right| left.0.cmp(right.0)); + if expected.windows(2).any(|pair| pair[0].0 == pair[1].0) { + return Err(PersistenceError::Storage(format!( + "duplicate spec batch entity type for tenant {tenant}" + ))); + } + let mut tx = self.pool().begin().await.map_err(storage_error)?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended('spec-catalog:' || $1, 0))") + .bind(tenant) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + for (entity_type, content_hash, csdl_xml) in expected { + let result = sqlx::query( + "WITH staged AS ( \ + DELETE FROM staged_specs \ + WHERE tenant = $1 AND entity_type = $2 AND content_hash = $3 \ + AND csdl_xml IS NOT DISTINCT FROM $4 \ + RETURNING * \ + ) \ + INSERT INTO specs \ + (tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, \ + verified, verification_status, updated_at) \ + SELECT tenant, entity_type, ioa_source, csdl_xml, content_hash, true, version, \ + false, 'pending', now() \ + FROM staged \ + ON CONFLICT (tenant, entity_type) DO UPDATE SET \ + ioa_source = EXCLUDED.ioa_source, csdl_xml = EXCLUDED.csdl_xml, \ + content_hash = EXCLUDED.content_hash, committed = true, \ + version = specs.version + 1, verified = false, \ + verification_status = 'pending', levels_passed = NULL, \ + levels_total = NULL, verification_result = NULL, updated_at = now()", + ) .bind(tenant) .bind(entity_type) - .execute(self.pool()) + .bind(content_hash) + .bind(csdl_xml) + .execute(&mut *tx) .await .map_err(storage_error)?; + if result.rows_affected() != 1 { + return Err(PersistenceError::Storage(format!( + "staged spec fingerprint changed for {tenant}/{entity_type}" + ))); + } + } + tx.commit().await.map_err(storage_error)?; Ok(()) } - pub async fn commit_specs(&self, tenant: &str) -> Result<(), PersistenceError> { - crate::dbm::postgres_query!( - "UPDATE specs SET committed = true, updated_at = now() WHERE tenant = $1" + /// Atomically persist verification and commit only the expected spec bytes. + pub async fn commit_verified_spec( + &self, + tenant: &str, + entity_type: &str, + expected_content_hash: &str, + expected_csdl_xml: &str, + update: PostgresSpecVerificationUpdate<'_>, + ) -> Result<(), PersistenceError> { + let verification_result = parse_optional_json(update.verification_result_json)?; + let mut tx = self.pool().begin().await.map_err(storage_error)?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended('spec-catalog:' || $1, 0))") + .bind(tenant) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + let rows: Vec<(i64,)> = sqlx::query_as( + "WITH staged AS ( \ + DELETE FROM staged_specs \ + WHERE tenant = $1 AND entity_type = $2 AND content_hash = $3 \ + AND csdl_xml IS NOT DISTINCT FROM $4 \ + RETURNING * \ + ), published AS ( \ + INSERT INTO specs \ + (tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, verified, \ + verification_status, levels_passed, levels_total, verification_result, updated_at) \ + SELECT tenant, entity_type, ioa_source, csdl_xml, content_hash, true, version, $6, \ + $5, $7, $8, $9, now() \ + FROM staged \ + ON CONFLICT (tenant, entity_type) DO UPDATE SET \ + ioa_source = EXCLUDED.ioa_source, csdl_xml = EXCLUDED.csdl_xml, \ + content_hash = EXCLUDED.content_hash, committed = true, \ + version = specs.version + 1, verified = EXCLUDED.verified, \ + verification_status = EXCLUDED.verification_status, \ + levels_passed = EXCLUDED.levels_passed, levels_total = EXCLUDED.levels_total, \ + verification_result = EXCLUDED.verification_result, updated_at = now() \ + RETURNING 1 \ + ) SELECT COUNT(*)::bigint FROM published" ) .bind(tenant) - .execute(self.pool()) + .bind(entity_type) + .bind(expected_content_hash) + .bind(expected_csdl_xml) + .bind(update.status) + .bind(update.verified) + .bind(update.levels_passed) + .bind(update.levels_total) + .bind(verification_result) + .fetch_all(&mut *tx) .await .map_err(storage_error)?; + if rows.first().map(|row| row.0) != Some(1) { + return Err(PersistenceError::Storage(format!( + "staged spec fingerprint changed for {tenant}/{entity_type}" + ))); + } + tx.commit().await.map_err(storage_error)?; Ok(()) } pub async fn delete_uncommitted_specs(&self) -> Result { - let result = crate::dbm::postgres_query!("DELETE FROM specs WHERE committed = false") - .execute(self.pool()) - .await - .map_err(storage_error)?; - Ok(result.rows_affected() as usize) + let rows: Vec<(i64,)> = crate::dbm::postgres_query_as!( + "WITH staged AS (DELETE FROM staged_specs RETURNING 1), \ + legacy AS (DELETE FROM specs WHERE committed = false RETURNING 1) \ + SELECT (SELECT COUNT(*) FROM staged) + (SELECT COUNT(*) FROM legacy)" + ) + .fetch_all(self.pool()) + .await + .map_err(storage_error)?; + Ok(rows.first().map(|row| row.0).unwrap_or_default() as usize) } pub async fn load_verification_cache( @@ -1127,7 +1264,8 @@ impl PostgresEventStore { tenant: &str, ) -> Result, PersistenceError> { let rows: Vec<(String, String, bool)> = crate::dbm::postgres_query_as!( - "SELECT entity_type, content_hash, verified FROM specs WHERE tenant = $1", + "SELECT entity_type, content_hash, verified FROM specs \ + WHERE tenant = $1 AND committed = true", ) .bind(tenant) .fetch_all(self.pool()) diff --git a/crates/temper-store-postgres/src/spec_catalog.rs b/crates/temper-store-postgres/src/spec_catalog.rs new file mode 100644 index 000000000..ad3ca3116 --- /dev/null +++ b/crates/temper-store-postgres/src/spec_catalog.rs @@ -0,0 +1,285 @@ +use std::collections::BTreeSet; + +use temper_runtime::persistence::PersistenceError; + +use crate::PostgresEventStore; + +impl PostgresEventStore { + /// Atomically publish a verified staged catalog under the replacement lock. + /// + /// Every incoming row must still match the staged content hash and CSDL. + /// The transaction promotes all rows, applies replacement omissions and + /// constraints, and marks the promoted catalog verified as one operation. + pub async fn persist_verified_spec_catalog_update( + &self, + tenant: &str, + expected: &[(&str, &str, &str)], + additional_removed_entity_types: &[String], + replace: bool, + cross_invariants_toml: Option<&str>, + ) -> Result, PersistenceError> { + let mut expected = expected.to_vec(); + expected.sort_unstable_by(|left, right| left.0.cmp(right.0)); + if expected.windows(2).any(|pair| pair[0].0 == pair[1].0) { + return Err(PersistenceError::Storage(format!( + "duplicate verified catalog entity type for tenant {tenant}" + ))); + } + + let mut tx = self + .pool() + .begin() + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + sqlx::query( + "SELECT pg_advisory_xact_lock( \ + hashtextextended('spec-catalog:' || $1, 0) \ + )", + ) + .bind(tenant) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + + let incoming = expected + .iter() + .map(|(entity_type, _, _)| *entity_type) + .collect::>(); + let mut removed_entity_types = if replace { + sqlx::query_scalar::<_, String>( + "SELECT entity_type FROM specs WHERE tenant = $1 \ + UNION \ + SELECT entity_type FROM staged_specs WHERE tenant = $1 \ + UNION \ + SELECT entity_type FROM spec_declaration_authority \ + WHERE tenant = $1 AND present = true \ + ORDER BY entity_type", + ) + .bind(tenant) + .fetch_all(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))? + .into_iter() + .filter(|entity_type| !incoming.contains(entity_type.as_str())) + .collect::>() + } else { + BTreeSet::new() + }; + removed_entity_types.extend( + additional_removed_entity_types + .iter() + .filter(|entity_type| !incoming.contains(entity_type.as_str())) + .cloned(), + ); + let removed_entity_types = removed_entity_types.into_iter().collect::>(); + + for (entity_type, content_hash, csdl_xml) in expected { + let result = sqlx::query( + "WITH staged AS ( \ + DELETE FROM staged_specs \ + WHERE tenant = $1 AND entity_type = $2 AND content_hash = $3 \ + AND csdl_xml IS NOT DISTINCT FROM $4 \ + RETURNING * \ + ) \ + INSERT INTO specs \ + (tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, \ + verified, verification_status, updated_at) \ + SELECT tenant, entity_type, ioa_source, csdl_xml, content_hash, true, version, \ + true, 'passed', now() \ + FROM staged \ + ON CONFLICT (tenant, entity_type) DO UPDATE SET \ + ioa_source = EXCLUDED.ioa_source, csdl_xml = EXCLUDED.csdl_xml, \ + content_hash = EXCLUDED.content_hash, committed = true, \ + version = specs.version + 1, verified = true, \ + verification_status = 'passed', levels_passed = NULL, \ + levels_total = NULL, verification_result = NULL, updated_at = now()", + ) + .bind(tenant) + .bind(entity_type) + .bind(content_hash) + .bind(csdl_xml) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + if result.rows_affected() != 1 { + return Err(PersistenceError::Storage(format!( + "staged spec fingerprint changed for {tenant}/{entity_type}" + ))); + } + } + for entity_type in &removed_entity_types { + sqlx::query("DELETE FROM staged_specs WHERE tenant = $1 AND entity_type = $2") + .bind(tenant) + .bind(entity_type) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + sqlx::query("SELECT tombstone_spec_declaration_authority($1, $2)") + .bind(tenant) + .bind(entity_type) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + } + if let Some(source) = cross_invariants_toml { + sqlx::query( + "INSERT INTO tenant_constraints \ + (tenant, cross_invariants_toml, version, updated_at) \ + VALUES ($1, $2, 1, now()) \ + ON CONFLICT(tenant) DO UPDATE SET \ + cross_invariants_toml = EXCLUDED.cross_invariants_toml, \ + version = tenant_constraints.version + 1, updated_at = now()", + ) + .bind(tenant) + .bind(source) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + } else if replace { + sqlx::query("DELETE FROM tenant_constraints WHERE tenant = $1") + .bind(tenant) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + } + tx.commit() + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + Ok(removed_entity_types) + } + + /// Atomically publish a tenant catalog update under the shared replacement lock. + /// + /// When `replace` is true, omissions are discovered after the tenant-scoped + /// transaction lock is acquired and tombstoned in the same transaction. Two + /// replicas therefore serialize source-of-truth replacements instead of + /// committing their union from stale pre-transaction snapshots. An omitted + /// constraint source is preserved for merges and cleared for replacements. + pub async fn persist_spec_catalog_update( + &self, + tenant: &str, + specs: &[(&str, &str, &str)], + csdl_xml: &str, + additional_removed_entity_types: &[String], + replace: bool, + cross_invariants_toml: Option<&str>, + ) -> Result, PersistenceError> { + let mut tx = self + .pool() + .begin() + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + sqlx::query( + "SELECT pg_advisory_xact_lock( \ + hashtextextended('spec-catalog:' || $1, 0) \ + )", + ) + .bind(tenant) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + + let incoming = specs + .iter() + .map(|(entity_type, _, _)| *entity_type) + .collect::>(); + let mut removed_entity_types = if replace { + sqlx::query_scalar::<_, String>( + "SELECT entity_type FROM specs WHERE tenant = $1 \ + UNION \ + SELECT entity_type FROM staged_specs WHERE tenant = $1 \ + UNION \ + SELECT entity_type FROM spec_declaration_authority \ + WHERE tenant = $1 AND present = true \ + ORDER BY entity_type", + ) + .bind(tenant) + .fetch_all(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))? + .into_iter() + .filter(|entity_type| !incoming.contains(entity_type.as_str())) + .collect::>() + } else { + BTreeSet::new() + }; + removed_entity_types.extend( + additional_removed_entity_types + .iter() + .filter(|entity_type| !incoming.contains(entity_type.as_str())) + .cloned(), + ); + let removed_entity_types = removed_entity_types.into_iter().collect::>(); + + for (entity_type, ioa_source, content_hash) in specs { + sqlx::query("DELETE FROM staged_specs WHERE tenant = $1 AND entity_type = $2") + .bind(tenant) + .bind(entity_type) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + sqlx::query( + "INSERT INTO specs \ + (tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, verified, verification_status, updated_at) \ + VALUES ($1, $2, $3, $4, $5, true, 1, false, 'pending', now()) \ + ON CONFLICT (tenant, entity_type) DO UPDATE SET \ + ioa_source = EXCLUDED.ioa_source, csdl_xml = EXCLUDED.csdl_xml, \ + content_hash = EXCLUDED.content_hash, committed = true, \ + version = specs.version + 1, verified = false, \ + verification_status = 'pending', levels_passed = NULL, \ + levels_total = NULL, verification_result = NULL, updated_at = now()", + ) + .bind(tenant) + .bind(entity_type) + .bind(ioa_source) + .bind(csdl_xml) + .bind(content_hash) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + } + for entity_type in &removed_entity_types { + sqlx::query("DELETE FROM staged_specs WHERE tenant = $1 AND entity_type = $2") + .bind(tenant) + .bind(entity_type) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + sqlx::query("SELECT tombstone_spec_declaration_authority($1, $2)") + .bind(tenant) + .bind(entity_type) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + } + if let Some(source) = cross_invariants_toml { + sqlx::query( + "INSERT INTO tenant_constraints \ + (tenant, cross_invariants_toml, version, updated_at) \ + VALUES ($1, $2, 1, now()) \ + ON CONFLICT (tenant) DO UPDATE SET \ + cross_invariants_toml = EXCLUDED.cross_invariants_toml, \ + version = tenant_constraints.version + 1, updated_at = now()", + ) + .bind(tenant) + .bind(source) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + } else if replace { + sqlx::query("DELETE FROM tenant_constraints WHERE tenant = $1") + .bind(tenant) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + } + tx.commit() + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + Ok(removed_entity_types) + } +} + +#[cfg(test)] +#[path = "spec_catalog_test.rs"] +mod tests; diff --git a/crates/temper-store-postgres/src/spec_catalog_test.rs b/crates/temper-store-postgres/src/spec_catalog_test.rs new file mode 100644 index 000000000..b17bafd3b --- /dev/null +++ b/crates/temper-store-postgres/src/spec_catalog_test.rs @@ -0,0 +1,223 @@ +use super::*; +use crate::PostgresSpecVerificationUpdate; +use crate::migration::run_migrations; + +#[test] +fn replacement_enumeration_includes_staged_only_entity_types() { + let Ok(database_url) = std::env::var("DATABASE_URL") else { + tracing::warn!("skipping Postgres integration test: DATABASE_URL is not set"); + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-staged-enumeration-{}", uuid::Uuid::new_v4()); + + store + .upsert_spec( + &tenant, + "StagedOnly", + "[automaton]\nname = \"StagedOnly\"\n", + "", + "staged-only-fingerprint", + ) + .await + .expect("stage catalog-only type"); + + assert_eq!( + store + .spec_replacement_entity_types(&tenant) + .await + .expect("enumerate replacement types"), + vec!["StagedOnly".to_string()] + ); + }); +} + +#[test] +fn verified_promotion_serializes_with_catalog_replacement() { + let Ok(database_url) = std::env::var("DATABASE_URL") else { + tracing::warn!("skipping Postgres integration test: DATABASE_URL is not set"); + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("tenant-promotion-lock-{}", uuid::Uuid::new_v4()); + let csdl = ""; + store + .upsert_spec( + &tenant, + "Item", + "[automaton]\nname = \"Item\"\n", + csdl, + "fingerprint", + ) + .await + .expect("stage Item"); + + let mut blocker = pool.begin().await.expect("begin blocker"); + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended('spec-catalog:' || $1, 0))") + .bind(&tenant) + .execute(&mut *blocker) + .await + .expect("acquire replacement lock"); + let promotion_tenant = tenant.clone(); + let mut promotion = sqlx::__rt::spawn(async move { + store + .commit_verified_spec( + &promotion_tenant, + "Item", + "fingerprint", + csdl, + PostgresSpecVerificationUpdate { + status: "passed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + }); + assert!( + sqlx::__rt::timeout(std::time::Duration::from_millis(100), &mut promotion) + .await + .is_err(), + "verified promotion must wait for the catalog replacement lock" + ); + blocker.commit().await.expect("release replacement lock"); + promotion.await.expect("promotion after lock release"); + }); +} + +#[test] +fn concurrent_replica_replacements_commit_one_complete_catalog() { + let Ok(database_url) = std::env::var("DATABASE_URL") else { + tracing::warn!("skipping Postgres integration test: DATABASE_URL is not set"); + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store_a = PostgresEventStore::new(pool.clone()); + let store_b = PostgresEventStore::new(pool.clone()); + let reader = PostgresEventStore::new(pool.clone()); + let tenant = format!("tenant-concurrent-catalog-{}", uuid::Uuid::new_v4()); + let csdl = ""; + let source_a = "[automaton]\nname = \"ItemA\"\n"; + let source_b = "[automaton]\nname = \"ItemB\"\n"; + let tenant_a = tenant.clone(); + let replacement_a = sqlx::__rt::spawn(async move { + let specs = [("ItemA", source_a, "fingerprint-a")]; + store_a + .persist_spec_catalog_update(&tenant_a, &specs, csdl, &[], true, None) + .await + }); + let tenant_b = tenant.clone(); + let replacement_b = sqlx::__rt::spawn(async move { + let specs = [("ItemB", source_b, "fingerprint-b")]; + store_b + .persist_spec_catalog_update(&tenant_b, &specs, csdl, &[], true, None) + .await + }); + replacement_a + .await + .expect("first replica replacement must commit"); + replacement_b + .await + .expect("second replica replacement must commit"); + + let committed: Vec = crate::dbm::postgres_query_scalar!( + "SELECT entity_type FROM specs \ + WHERE tenant = $1 AND committed = true ORDER BY entity_type", + ) + .bind(&tenant) + .fetch_all(&pool) + .await + .expect("load committed catalog"); + assert!( + committed == ["ItemA"] || committed == ["ItemB"], + "the final durable catalog must be one serialized replacement, got {committed:?}" + ); + assert_eq!( + reader + .spec_replacement_entity_types(&tenant) + .await + .expect("load present authority"), + committed, + "the authority rows must recover the same single catalog" + ); + }); +} + +#[test] +fn merge_without_constraints_preserves_them_across_restart_and_replace_clears_them() { + let Ok(database_url) = std::env::var("DATABASE_URL") else { + tracing::warn!("skipping Postgres integration test: DATABASE_URL is not set"); + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("tenant-merge-constraints-{}", uuid::Uuid::new_v4()); + let csdl = ""; + let source_a = "[automaton]\nname = \"ItemA\"\n"; + let source_b = "[automaton]\nname = \"ItemB\"\n"; + let specs_a = [("ItemA", source_a, "fingerprint-a")]; + let specs_b = [("ItemB", source_b, "fingerprint-b")]; + let constraints = r#"version = 1 +default_delete_policy = "restrict" + +[[invariant]] +name = "payment_must_be_captured" +kind = "hard" +on = "Order.Submit" +assert = 'related(Payment, payment_id).status in ["Captured"]' +"#; + + store + .persist_spec_catalog_update(&tenant, &specs_a, csdl, &[], true, Some(constraints)) + .await + .expect("seed replacement with constraints"); + store + .persist_spec_catalog_update(&tenant, &specs_b, csdl, &[], false, None) + .await + .expect("merge without constraints"); + drop(store); + drop(pool); + + let reopened_pool = sqlx::PgPool::connect(&database_url) + .await + .expect("reconnect after merge"); + let preserved: String = crate::dbm::postgres_query_scalar!( + "SELECT cross_invariants_toml FROM tenant_constraints WHERE tenant = $1", + ) + .bind(&tenant) + .fetch_one(&reopened_pool) + .await + .expect("constraints must survive merge restart"); + assert_eq!(preserved, constraints); + + let reopened = PostgresEventStore::new(reopened_pool.clone()); + reopened + .persist_spec_catalog_update(&tenant, &specs_a, csdl, &[], true, None) + .await + .expect("constraint-free replacement"); + let cleared: Option = crate::dbm::postgres_query_scalar!( + "SELECT cross_invariants_toml FROM tenant_constraints WHERE tenant = $1", + ) + .bind(&tenant) + .fetch_optional(&reopened_pool) + .await + .expect("read cleared constraints"); + assert_eq!(cleared, None); + }); +} diff --git a/crates/temper-store-postgres/src/store.rs b/crates/temper-store-postgres/src/store.rs index 21aa5c717..c79acfe0c 100644 --- a/crates/temper-store-postgres/src/store.rs +++ b/crates/temper-store-postgres/src/store.rs @@ -6,7 +6,8 @@ use std::time::Instant; -use sqlx::{Acquire, PgPool}; +use sha2::{Digest, Sha256}; +use sqlx::{Acquire, PgPool, Postgres, Transaction}; use temper_runtime::persistence::{ EntityVectorCandidate, EntityVectorRow, EventMetadata, EventStore, PersistenceAppend, PersistenceAppendResult, PersistenceEnvelope, PersistenceError, pack_f32_le, unpack_f32_le, @@ -20,6 +21,13 @@ use crate::metrics::{ use crate::segments; const EVENT_APPEND_OPERATION: &str = "event_append"; +const ABSENT_DECLARATION_FINGERPRINT: &str = "absent:v1"; + +fn spec_content_fingerprint(ioa_source: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(ioa_source.as_bytes()); + format!("{:x}", hasher.finalize()) +} /// A PostgreSQL-backed event store. /// @@ -41,6 +49,258 @@ impl PostgresEventStore { pub fn pool(&self) -> &PgPool { &self.pool } + + /// Entity types that a source-of-truth replacement must account for. + /// + /// Includes uncommitted catalog rows and compatibility authority created + /// without a catalog row. + pub async fn spec_replacement_entity_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + crate::dbm::postgres_query_scalar!( + "SELECT entity_type FROM specs WHERE tenant = $1 \ + UNION \ + SELECT entity_type FROM staged_specs WHERE tenant = $1 \ + UNION \ + SELECT entity_type FROM spec_declaration_authority \ + WHERE tenant = $1 AND present = true \ + ORDER BY entity_type", + ) + .bind(tenant) + .fetch_all(&self.pool) + .await + .map_err(|error| PersistenceError::Storage(error.to_string())) + } + + async fn spec_declaration_authority_with_barrier( + tx: &mut Transaction<'_, Postgres>, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + // This lock is held through commit. Spec mutation takes an exclusive lock + // on the same authority row, so a writer cannot validate declaration A and + // then co-commit A-derived vector rows after declaration B becomes durable. + let authority: Option<(i64, String, String, bool)> = crate::dbm::postgres_query_as!( + "SELECT revision, ioa_source, declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = $2 FOR SHARE", + ) + .bind(tenant) + .bind(entity_type) + .fetch_optional(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + let Some((revision, ioa_source, declaration_fingerprint, present)) = authority else { + return Ok(None); + }; + let revision = u64::try_from(revision).map_err(|_| { + PersistenceError::Storage(format!( + "invalid durable spec revision for {tenant}:{entity_type}" + )) + })?; + let fingerprint = if present { + if declaration_fingerprint.is_empty() { + spec_content_fingerprint(&ioa_source) + } else { + declaration_fingerprint + } + } else { + ABSENT_DECLARATION_FINGERPRINT.to_string() + }; + Ok(Some((revision, fingerprint))) + } + + async fn bootstrap_live_spec_declaration_if_absent( + tx: &mut Transaction<'_, Postgres>, + tenant: &str, + entity_type: &str, + supplied_fingerprint: &str, + ) -> Result<(), PersistenceError> { + // Compatibility constructors can have an in-memory transition table but + // no persisted spec catalog. The per-type transaction lock serializes + // racing first writers: one inserts its fingerprint, while every loser + // observes and validates against that committed winner below. A catalog + // row or retained tombstone prevents this insert and cannot be overwritten. + crate::dbm::postgres_query!( + "SELECT pg_advisory_xact_lock( \ + hashtextextended($1 || ':' || $2, 0) \ + )", + ) + .bind(tenant) + .bind(entity_type) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + crate::dbm::postgres_query!( + "INSERT INTO spec_declaration_authority \ + (tenant, entity_type, revision, ioa_source, declaration_fingerprint, present) \ + SELECT $1, $2, 1, '', $3, true \ + WHERE NOT EXISTS ( \ + SELECT 1 FROM specs WHERE tenant = $1 AND entity_type = $2 \ + ) \ + ON CONFLICT (tenant, entity_type) DO NOTHING", + ) + .bind(tenant) + .bind(entity_type) + .bind(supplied_fingerprint) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(()) + } + + async fn spec_declaration_with_compat_bootstrap( + tx: &mut Transaction<'_, Postgres>, + tenant: &str, + entity_type: &str, + supplied_fingerprint: &str, + ) -> Result<(u64, String), PersistenceError> { + if let Some(authority) = + Self::spec_declaration_authority_with_barrier(tx, tenant, entity_type).await? + { + return Ok(authority); + } + + // Only the truly empty compatibility path takes the exclusive advisory + // lock. The insert rechecks catalog/authority after the lock is acquired; + // normal writers retain only the shared authority-row lock through commit. + Self::bootstrap_live_spec_declaration_if_absent( + tx, + tenant, + entity_type, + supplied_fingerprint, + ) + .await?; + Self::spec_declaration_authority_with_barrier(tx, tenant, entity_type) + .await? + .ok_or_else(|| { + PersistenceError::Storage(format!( + "missing durable spec declaration authority for {tenant}:{entity_type}" + )) + }) + } + + pub(crate) async fn validate_live_spec_declaration( + tx: &mut Transaction<'_, Postgres>, + tenant: &str, + entity_type: &str, + supplied_fingerprint: &str, + ) -> Result<(), PersistenceError> { + if supplied_fingerprint.is_empty() { + return Err(PersistenceError::Storage(format!( + "live append requires a nonempty spec declaration fingerprint for {tenant}:{entity_type}" + ))); + } + let (_, authoritative_fingerprint) = Self::spec_declaration_with_compat_bootstrap( + tx, + tenant, + entity_type, + supplied_fingerprint, + ) + .await?; + if authoritative_fingerprint != supplied_fingerprint { + return Err(PersistenceError::Storage(format!( + "stale spec declaration fingerprint for {tenant}:{entity_type}" + ))); + } + Ok(()) + } + + async fn current_vector_generation_with_barrier( + tx: &mut Transaction<'_, Postgres>, + tenant: &str, + entity_type: &str, + ) -> Result { + crate::dbm::postgres_query!( + "INSERT INTO entity_vector_reconciliation_generation \ + (tenant, entity_type, generation, vector_set) VALUES ($1, $2, 0, '') \ + ON CONFLICT (tenant, entity_type) DO NOTHING", + ) + .bind(tenant) + .bind(entity_type) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + let (generation,): (i64,) = crate::dbm::postgres_query_as!( + "SELECT generation FROM entity_vector_reconciliation_generation \ + WHERE tenant = $1 AND entity_type = $2 FOR SHARE", + ) + .bind(tenant) + .bind(entity_type) + .fetch_one(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(generation as u64) + } + + async fn reconcile_live_vector_rows( + tx: &mut Transaction<'_, Postgres>, + tenant: &str, + entity_type: &str, + entity_id: &str, + new_sequence: u64, + vector_rows: &[EntityVectorRow], + ) -> Result<(), PersistenceError> { + // SHARE lets independent same-type writers proceed together, while still + // conflicting with the generation row's UPDATE during a new reconciliation. + let generation = + Self::current_vector_generation_with_barrier(tx, tenant, entity_type).await?; + let applied: Option<(i64,)> = crate::dbm::postgres_query_as!( + "INSERT INTO entity_vector_index_version \ + (tenant, entity_type, entity_id, reconciliation_generation, sequence_nr) \ + VALUES ($1, $2, $3, $4, $5) \ + ON CONFLICT (tenant, entity_type, entity_id) DO UPDATE SET \ + reconciliation_generation = EXCLUDED.reconciliation_generation, \ + sequence_nr = EXCLUDED.sequence_nr \ + WHERE entity_vector_index_version.reconciliation_generation < EXCLUDED.reconciliation_generation \ + OR (entity_vector_index_version.reconciliation_generation = EXCLUDED.reconciliation_generation \ + AND entity_vector_index_version.sequence_nr <= EXCLUDED.sequence_nr) \ + RETURNING sequence_nr", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(generation as i64) + .bind(new_sequence as i64) + .fetch_optional(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + if applied.is_none() { + return Err(PersistenceError::Storage(format!( + "vector-index fence for {tenant}:{entity_type}:{entity_id} is ahead of live journal sequence {new_sequence} in reconciliation generation {generation}" + ))); + } + + crate::dbm::postgres_query!( + "DELETE FROM entity_vector_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + for row in vector_rows { + crate::dbm::postgres_query!( + "INSERT INTO entity_vector_index \ + (tenant, entity_type, decl_name, model_tag, entity_id, vector, sequence_nr) \ + VALUES ($1, $2, $3, $4, $5, $6, $7)", + ) + .bind(tenant) + .bind(entity_type) + .bind(&row.decl_name) + .bind(&row.model_tag) + .bind(entity_id) + .bind(pack_f32_le(&row.vector)) + .bind(new_sequence as i64) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + } + Ok(()) + } } // --------------------------------------------------------------------------- @@ -63,8 +323,16 @@ impl EventStore for PostgresEventStore { expected_sequence: u64, events: &[PersistenceEnvelope], ) -> Result { - self.append_with_index_rows(persistence_id, expected_sequence, events, &[], &[], false) - .await + self.append_with_index_rows( + persistence_id, + expected_sequence, + events, + &[], + &[], + false, + None, + ) + .await } async fn append_with_index_rows( @@ -75,6 +343,7 @@ impl EventStore for PostgresEventStore { key_rows: &[temper_runtime::persistence::EntityKeyRow], vector_rows: &[EntityVectorRow], reconcile_vectors: bool, + spec_declaration_fingerprint: Option<&str>, ) -> Result { let (tenant, entity_type, entity_id) = parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; @@ -119,6 +388,15 @@ impl EventStore for PostgresEventStore { } }; + if reconcile_vectors && spec_declaration_fingerprint.is_none() { + return Err(PersistenceError::Storage(format!( + "vector reconciliation append requires a spec declaration fingerprint for {tenant}:{entity_type}" + ))); + } + if let Some(fingerprint) = spec_declaration_fingerprint { + Self::validate_live_spec_declaration(&mut tx, tenant, entity_type, fingerprint).await?; + } + let row: Option<(i64,)> = crate::dbm::postgres_query_as!( "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", @@ -252,33 +530,15 @@ impl EventStore for PostgresEventStore { // the stale rows instead of leaving them to rank forever. No uniqueness // constraint; vectors are derived ranking state. if reconcile_vectors { - crate::dbm::postgres_query!( - "DELETE FROM entity_vector_index \ - WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + Self::reconcile_live_vector_rows( + &mut tx, + tenant, + entity_type, + entity_id, + new_seq, + vector_rows, ) - .bind(tenant) - .bind(entity_type) - .bind(entity_id) - .execute(&mut *tx) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - for row in vector_rows { - crate::dbm::postgres_query!( - "INSERT INTO entity_vector_index \ - (tenant, entity_type, decl_name, model_tag, entity_id, vector, sequence_nr) \ - VALUES ($1, $2, $3, $4, $5, $6, $7)", - ) - .bind(tenant) - .bind(entity_type) - .bind(&row.decl_name) - .bind(&row.model_tag) - .bind(entity_id) - .bind(pack_f32_le(&row.vector)) - .bind(new_seq as i64) - .execute(&mut *tx) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - } + .await?; } let commit_started = Instant::now(); @@ -468,21 +728,254 @@ impl EventStore for PostgresEventStore { Ok(row.map(|(id,)| id)) } + async fn begin_vector_index_reconciliation( + &self, + tenant: &str, + entity_type: &str, + vector_set: &str, + declaration_revision: u64, + declaration_fingerprint: &str, + ) -> Result { + if declaration_revision == 0 || declaration_fingerprint.is_empty() { + return Err(PersistenceError::Storage(format!( + "vector declaration revision must be nonzero and fingerprinted for {tenant}:{entity_type}" + ))); + } + let mut tx = self + .pool + .begin() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + + // The authority row survives hard spec deletion. Its trigger advances the + // revision and fences existing work in the same transaction as every IOA + // mutation, including delete/re-add. + let (authoritative_revision, stored_fingerprint) = + Self::spec_declaration_with_compat_bootstrap( + &mut tx, + tenant, + entity_type, + declaration_fingerprint, + ) + .await?; + if stored_fingerprint != declaration_fingerprint { + return Err(PersistenceError::Storage(format!( + "stale vector declaration fingerprint for {tenant}:{entity_type}" + ))); + } + let stored_revision = i64::try_from(authoritative_revision).map_err(|_| { + PersistenceError::Storage(format!( + "vector declaration revision exhausted for {tenant}:{entity_type}" + )) + })?; + + let inserted: Option<(i64,)> = crate::dbm::postgres_query_as!( + "INSERT INTO entity_vector_reconciliation_generation \ + (tenant, entity_type, generation, declaration_revision, declaration_fingerprint, vector_set) \ + VALUES ($1, $2, 1, $3, $4, $5) \ + ON CONFLICT (tenant, entity_type) DO NOTHING \ + RETURNING generation", + ) + .bind(tenant) + .bind(entity_type) + .bind(stored_revision) + .bind(declaration_fingerprint) + .bind(vector_set) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + if let Some((generation,)) = inserted { + crate::dbm::postgres_query!( + "DELETE FROM vector_index_backfill_watermark \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + tx.commit() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + return u64::try_from(generation).map_err(|_| { + PersistenceError::Storage(format!( + "invalid vector reconciliation generation for {tenant}:{entity_type}" + )) + }); + } + + let (generation, current_revision, current_fingerprint, current_set): ( + i64, + i64, + String, + String, + ) = crate::dbm::postgres_query_as!( + "SELECT generation, declaration_revision, declaration_fingerprint, vector_set \ + FROM entity_vector_reconciliation_generation \ + WHERE tenant = $1 AND entity_type = $2 FOR UPDATE", + ) + .bind(tenant) + .bind(entity_type) + .fetch_one(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + + let current_revision = u64::try_from(current_revision).map_err(|_| { + PersistenceError::Storage(format!( + "invalid vector declaration revision for {tenant}:{entity_type}" + )) + })?; + if authoritative_revision < current_revision { + return Err(PersistenceError::Storage(format!( + "vector reconciliation revision {current_revision} exceeds declaration authority {authoritative_revision} for {tenant}:{entity_type}" + ))); + } + if authoritative_revision == current_revision { + if current_fingerprint == declaration_fingerprint && current_set == vector_set { + tx.commit() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + return u64::try_from(generation).map_err(|_| { + PersistenceError::Storage(format!( + "invalid vector reconciliation generation for {tenant}:{entity_type}" + )) + }); + } + if !current_fingerprint.is_empty() || !current_set.is_empty() { + return Err(PersistenceError::Storage(format!( + "conflicting vector declaration at revision {authoritative_revision} for {tenant}:{entity_type}" + ))); + } + } + + // Spec triggers already advance the generation and leave an empty claim. + // The fallback increment covers upgraded generation-zero/live-write rows. + let next_generation = if authoritative_revision == current_revision { + generation + } else { + generation.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "vector reconciliation generation exhausted for {tenant}:{entity_type}" + )) + })? + }; + crate::dbm::postgres_query!( + "UPDATE entity_vector_reconciliation_generation \ + SET generation = $3, declaration_revision = $4, \ + declaration_fingerprint = $5, vector_set = $6 \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .bind(next_generation) + .bind(stored_revision) + .bind(declaration_fingerprint) + .bind(vector_set) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + // Claiming a trigger-advanced or upgraded revision withdraws any legacy + // completion claim. An exact retry returned above leaves it intact. + crate::dbm::postgres_query!( + "DELETE FROM vector_index_backfill_watermark \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + tx.commit() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + u64::try_from(next_generation).map_err(|_| { + PersistenceError::Storage(format!( + "invalid vector reconciliation generation for {tenant}:{entity_type}" + )) + }) + } + async fn backfill_entity_vectors( &self, tenant: &str, entity_type: &str, entity_id: &str, + reconciliation_generation: u64, + observed_sequence: u64, vector_rows: &[EntityVectorRow], ) -> Result<(), PersistenceError> { - // Reconcile: DELETE all of the entity's rows, then insert the current ones. - // Empty `vector_rows` purges the entity (deleted / un-embedded). Always runs - // the delete (even for empty rows) so a purge is honored. + if reconciliation_generation == 0 { + return Err(PersistenceError::Storage( + "vector reconciliation generation zero is reserved for pre-reconciliation live writes" + .to_string(), + )); + } + // Lock and validate the type generation in the same transaction as the + // entity replacement. Beginning a newer declaration set invalidates this + // work before it can mutate rows. let mut tx = self .pool .begin() .await .map_err(|e| PersistenceError::Storage(e.to_string()))?; + let current: Option<(i64,)> = crate::dbm::postgres_query_as!( + "SELECT generation FROM entity_vector_reconciliation_generation \ + WHERE tenant = $1 AND entity_type = $2 FOR SHARE", + ) + .bind(tenant) + .bind(entity_type) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + let current_generation = current.map(|(generation,)| generation as u64).unwrap_or(0); + if current_generation != reconciliation_generation { + return Err(PersistenceError::Storage(format!( + "stale vector reconciliation generation {reconciliation_generation} for {tenant}:{entity_type}; current generation is {current_generation}" + ))); + } + + let applied: Option<(i64, i64)> = crate::dbm::postgres_query_as!( + "INSERT INTO entity_vector_index_version \ + (tenant, entity_type, entity_id, reconciliation_generation, sequence_nr) \ + VALUES ($1, $2, $3, $4, $5) \ + ON CONFLICT (tenant, entity_type, entity_id) DO UPDATE SET \ + reconciliation_generation = EXCLUDED.reconciliation_generation, \ + sequence_nr = EXCLUDED.sequence_nr \ + WHERE entity_vector_index_version.reconciliation_generation < EXCLUDED.reconciliation_generation \ + OR (entity_vector_index_version.reconciliation_generation = EXCLUDED.reconciliation_generation \ + AND entity_vector_index_version.sequence_nr <= EXCLUDED.sequence_nr) \ + RETURNING reconciliation_generation, sequence_nr", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(reconciliation_generation as i64) + .bind(observed_sequence as i64) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + if applied.is_none() { + let fence: Option<(i64, i64)> = crate::dbm::postgres_query_as!( + "SELECT reconciliation_generation, sequence_nr \ + FROM entity_vector_index_version \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + if fence.is_some_and(|(generation, _)| generation as u64 > reconciliation_generation) { + return Err(PersistenceError::Storage(format!( + "vector-index fence generation is ahead of current type generation for {tenant}:{entity_type}:{entity_id}" + ))); + } + tx.commit() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + return Ok(()); + } crate::dbm::postgres_query!( "DELETE FROM entity_vector_index \ WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", @@ -497,7 +990,7 @@ impl EventStore for PostgresEventStore { crate::dbm::postgres_query!( "INSERT INTO entity_vector_index \ (tenant, entity_type, decl_name, model_tag, entity_id, vector, sequence_nr) \ - VALUES ($1, $2, $3, $4, $5, $6, 0)", + VALUES ($1, $2, $3, $4, $5, $6, $7)", ) .bind(tenant) .bind(entity_type) @@ -505,6 +998,7 @@ impl EventStore for PostgresEventStore { .bind(&row.model_tag) .bind(entity_id) .bind(pack_f32_le(&row.vector)) + .bind(observed_sequence as i64) .execute(&mut *tx) .await .map_err(|e| PersistenceError::Storage(e.to_string()))?; @@ -553,13 +1047,39 @@ impl EventStore for PostgresEventStore { &self, tenant: &str, entity_type: &str, + reconciliation_generation: u64, vector_set: &str, ) -> Result<(), PersistenceError> { - let mut conn = self + if reconciliation_generation == 0 { + return Err(PersistenceError::Storage( + "vector reconciliation generation zero cannot publish a watermark".to_string(), + )); + } + let mut tx = self .pool - .acquire() + .begin() .await .map_err(|e| PersistenceError::Storage(e.to_string()))?; + let current: Option<(i64, String)> = crate::dbm::postgres_query_as!( + "SELECT generation, vector_set FROM entity_vector_reconciliation_generation \ + WHERE tenant = $1 AND entity_type = $2 FOR SHARE", + ) + .bind(tenant) + .bind(entity_type) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + if current.as_ref().map(|(generation, signature)| { + *generation as u64 == reconciliation_generation && signature == vector_set + }) != Some(true) + { + let current_generation = current + .map(|(generation, _)| generation as u64) + .unwrap_or(0); + return Err(PersistenceError::Storage(format!( + "stale vector reconciliation generation {reconciliation_generation} for {tenant}:{entity_type}; current generation is {current_generation}" + ))); + } crate::dbm::postgres_query!( "INSERT INTO vector_index_backfill_watermark (tenant, entity_type, vector_set) \ VALUES ($1, $2, $3) \ @@ -569,9 +1089,12 @@ impl EventStore for PostgresEventStore { .bind(tenant) .bind(entity_type) .bind(vector_set) - .execute(&mut *conn) + .execute(&mut *tx) .await .map_err(|e| PersistenceError::Storage(e.to_string()))?; + tx.commit() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; Ok(()) } @@ -594,6 +1117,23 @@ impl EventStore for PostgresEventStore { Ok(rows.into_iter().collect()) } + async fn vector_reconciliation_entity_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + let rows: Vec<(String,)> = crate::dbm::postgres_query_as!( + "SELECT entity_type FROM entity_vector_reconciliation_generation WHERE tenant = $1 \ + UNION SELECT entity_type FROM entity_vector_index_version WHERE tenant = $1 \ + UNION SELECT entity_type FROM entity_vector_index WHERE tenant = $1 \ + ORDER BY entity_type", + ) + .bind(tenant) + .fetch_all(&self.pool) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(rows.into_iter().map(|(entity_type,)| entity_type).collect()) + } + async fn vectored_entity_ids_for_type( &self, tenant: &str, @@ -616,6 +1156,24 @@ impl EventStore for PostgresEventStore { Ok(rows.into_iter().map(|(entity_id,)| entity_id).collect()) } + async fn list_vector_repair_entity_ids( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + let rows: Vec<(String,)> = crate::dbm::postgres_query_as!( + "SELECT DISTINCT entity_id FROM events \ + WHERE tenant = $1 AND entity_type = $2 \ + ORDER BY entity_id", + ) + .bind(tenant) + .bind(entity_type) + .fetch_all(&self.pool) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(rows.into_iter().map(|(entity_id,)| entity_id).collect()) + } + /// Atomically append to multiple entity journals in one PostgreSQL /// transaction. Used as the storage foundation for cross-actor Composite /// transactions: every stream's optimistic-concurrency check must pass @@ -629,6 +1187,7 @@ impl EventStore for PostgresEventStore { } let mut seen = std::collections::BTreeSet::new(); + let mut declaration_fingerprints = std::collections::BTreeMap::new(); for append in appends { if !seen.insert(append.persistence_id.as_str()) { return Err(PersistenceError::Storage(format!( @@ -636,6 +1195,25 @@ impl EventStore for PostgresEventStore { append.persistence_id ))); } + if append.reconcile_vectors && append.spec_declaration_fingerprint.is_none() { + return Err(PersistenceError::Storage(format!( + "vector reconciliation append requires a spec declaration fingerprint for '{}'", + append.persistence_id + ))); + } + if let Some(fingerprint) = append.spec_declaration_fingerprint.as_deref() { + let (tenant, entity_type, _) = parse_persistence_id_parts(&append.persistence_id) + .map_err(PersistenceError::Storage)?; + let key = (tenant.to_string(), entity_type.to_string()); + if let Some(existing) = declaration_fingerprints.get(&key) + && existing != fingerprint + { + return Err(PersistenceError::Storage(format!( + "conflicting spec declaration fingerprints in append_batch for {tenant}:{entity_type}" + ))); + } + declaration_fingerprints.insert(key, fingerprint.to_string()); + } } let mut transaction_timer = PostgresTransactionTimer::start(EVENT_APPEND_OPERATION); @@ -678,6 +1256,13 @@ impl EventStore for PostgresEventStore { } }; + // Lock authority rows in deterministic tenant/type order before checking + // or mutating any journal. Keeping these SHARE locks through commit makes + // the complete batch atomic with respect to spec declaration changes. + for ((tenant, entity_type), fingerprint) in &declaration_fingerprints { + Self::validate_live_spec_declaration(&mut tx, tenant, entity_type, fingerprint).await?; + } + let mut parsed = Vec::with_capacity(appends.len()); for append in appends { let (tenant, entity_type, entity_id) = @@ -765,6 +1350,17 @@ impl EventStore for PostgresEventStore { ) .await?; } + if append.reconcile_vectors { + Self::reconcile_live_vector_rows( + &mut tx, + tenant, + entity_type, + entity_id, + new_seq, + &append.vector_rows, + ) + .await?; + } results.push(PersistenceAppendResult { persistence_id: append.persistence_id.clone(), sequence_nr: new_seq, @@ -1043,6 +1639,14 @@ impl EventStore for PostgresEventStore { #[path = "store_projection_test.rs"] mod projection_tests; +#[cfg(test)] +#[path = "store_declaration_authority_test.rs"] +mod declaration_authority_tests; + +#[cfg(test)] +#[path = "store_vector_reconciliation_test.rs"] +mod vector_reconciliation_tests; + #[cfg(test)] mod tests { use super::*; @@ -1121,7 +1725,7 @@ mod tests { let database_url = match std::env::var("DATABASE_URL") { Ok(url) => url, Err(_) => { - eprintln!("skipping Postgres integration test: DATABASE_URL is not set"); + tracing::warn!("skipping Postgres integration test: DATABASE_URL is not set"); return; } }; @@ -1340,7 +1944,7 @@ mod tests { let database_url = match std::env::var("DATABASE_URL") { Ok(url) => url, Err(_) => { - eprintln!("skipping Postgres integration test: DATABASE_URL is not set"); + tracing::warn!("skipping Postgres integration test: DATABASE_URL is not set"); return; } }; diff --git a/crates/temper-store-postgres/src/store_declaration_authority_test.rs b/crates/temper-store-postgres/src/store_declaration_authority_test.rs new file mode 100644 index 000000000..b9b31a07c --- /dev/null +++ b/crates/temper-store-postgres/src/store_declaration_authority_test.rs @@ -0,0 +1,848 @@ +use std::time::Duration; + +use super::*; +use crate::migration::run_migrations; + +fn database_url(test_name: &str) -> Option { + match std::env::var("DATABASE_URL") { + Ok(url) => Some(url), + Err(_) => { + tracing::warn!( + test_name, + "skipping Postgres integration test: DATABASE_URL is not set" + ); + None + } + } +} + +#[test] +fn verified_commit_rejects_same_ioa_with_replaced_csdl() { + let Some(database_url) = database_url("verified_commit_rejects_same_ioa_with_replaced_csdl") + else { + return; + }; + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-csdl-commit-{}", uuid::Uuid::new_v4()); + let ioa = "[automaton]\nname = \"Item\"\n"; + let fingerprint = spec_content_fingerprint(ioa); + let csdl_a = ""; + let csdl_b = ""; + + store + .upsert_spec(&tenant, "Item", ioa, csdl_a, &fingerprint) + .await + .expect("stage verified pair A"); + store + .commit_verified_spec( + &tenant, + "Item", + &fingerprint, + csdl_a, + crate::PostgresSpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .expect("commit verified pair A"); + store + .upsert_spec(&tenant, "Item", ioa, csdl_b, &fingerprint) + .await + .expect("stage CSDL B"); + + store + .commit_verified_spec( + &tenant, + "Item", + &fingerprint, + csdl_a, + crate::PostgresSpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .expect_err("verification of CSDL A must not publish staged CSDL B"); + + let committed: (String, bool) = crate::dbm::postgres_query_as!( + "SELECT csdl_xml, verified FROM specs \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read committed CSDL A"); + assert_eq!(committed, (csdl_a.to_string(), true)); + let staged: (String,) = crate::dbm::postgres_query_as!( + "SELECT csdl_xml FROM staged_specs \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read staged CSDL B"); + assert_eq!(staged.0, csdl_b); + }); +} + +#[test] +fn scoped_commit_does_not_promote_unrelated_staging() { + let Some(database_url) = database_url("scoped_commit_does_not_promote_unrelated_staging") + else { + return; + }; + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-scoped-commit-{}", uuid::Uuid::new_v4()); + let csdl = ""; + let item = "[automaton]\nname = \"Item\"\n"; + let unrelated = "[automaton]\nname = \"Unrelated\"\n"; + let item_fingerprint = spec_content_fingerprint(item); + let unrelated_fingerprint = spec_content_fingerprint(unrelated); + + store + .upsert_spec(&tenant, "Item", item, csdl, &item_fingerprint) + .await + .expect("stage owned spec"); + store + .upsert_spec( + &tenant, + "Unrelated", + unrelated, + csdl, + &unrelated_fingerprint, + ) + .await + .expect("stage unrelated spec"); + store + .commit_verified_spec( + &tenant, + "Item", + &item_fingerprint, + csdl, + crate::PostgresSpecVerificationUpdate { + status: "pending", + verified: false, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .expect("commit only owned spec"); + + let committed = store + .load_specs() + .await + .expect("load committed specs") + .into_iter() + .filter(|row| row.tenant == tenant) + .collect::>(); + assert_eq!(committed.len(), 1); + assert_eq!(committed[0].entity_type, "Item"); + let unrelated_staged: (String,) = crate::dbm::postgres_query_as!( + "SELECT content_hash FROM staged_specs \ + WHERE tenant = $1 AND entity_type = 'Unrelated'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("unrelated staging remains quarantined"); + assert_eq!(unrelated_staged.0, unrelated_fingerprint); + }); +} + +#[test] +fn spec_batch_commit_rolls_back_every_promotion_on_mismatch() { + let Some(database_url) = + database_url("spec_batch_commit_rolls_back_every_promotion_on_mismatch") + else { + return; + }; + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-batch-rollback-{}", uuid::Uuid::new_v4()); + let csdl = ""; + let item = "[automaton]\nname = \"Item\"\n"; + let issue = "[automaton]\nname = \"Issue\"\n"; + let item_hash = spec_content_fingerprint(item); + let issue_hash = spec_content_fingerprint(issue); + + store + .upsert_spec(&tenant, "Item", item, csdl, &item_hash) + .await + .expect("stage Item"); + store + .upsert_spec(&tenant, "Issue", issue, csdl, &issue_hash) + .await + .expect("stage Issue"); + store + .commit_spec_batch( + &tenant, + &[ + ("Item", item_hash.as_str(), csdl), + ("Issue", "wrong-hash", csdl), + ], + ) + .await + .expect_err("one mismatch must roll back the whole batch"); + + let committed: (i64,) = + crate::dbm::postgres_query_as!("SELECT COUNT(*) FROM specs WHERE tenant = $1") + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("count committed rows"); + let staged: (i64,) = + crate::dbm::postgres_query_as!("SELECT COUNT(*) FROM staged_specs WHERE tenant = $1") + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("count staged rows"); + assert_eq!(committed.0, 0); + assert_eq!(staged.0, 2); + }); +} + +#[test] +fn deletion_fences_a_stale_verifier_from_resurrecting_staging() { + let Some(database_url) = + database_url("deletion_fences_a_stale_verifier_from_resurrecting_staging") + else { + return; + }; + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-delete-fence-{}", uuid::Uuid::new_v4()); + let ioa = "[automaton]\nname = \"Item\"\n"; + let csdl = ""; + let fingerprint = spec_content_fingerprint(ioa); + + store + .upsert_spec(&tenant, "Item", ioa, csdl, &fingerprint) + .await + .expect("stage spec"); + store + .delete_spec(&tenant, "Item") + .await + .expect("delete declaration and staging"); + store + .commit_verified_spec( + &tenant, + "Item", + &fingerprint, + csdl, + crate::PostgresSpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .expect_err("stale verification must not resurrect a deleted spec"); + + let catalog_count: (i64,) = crate::dbm::postgres_query_as!( + "SELECT COUNT(*) FROM specs WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("count committed rows"); + let staged_count: (i64,) = crate::dbm::postgres_query_as!( + "SELECT COUNT(*) FROM staged_specs WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("count staged rows"); + assert_eq!(catalog_count.0, 0); + assert_eq!(staged_count.0, 0); + }); +} + +fn test_envelope(event_type: &str) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: 0, + event_type: event_type.to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: uuid::Uuid::new_v4(), + causation_id: uuid::Uuid::new_v4(), + correlation_id: uuid::Uuid::new_v4(), + timestamp: chrono::Utc::now(), + actor_id: "authority-test".to_string(), + }, + } +} + +#[test] +fn fresh_writers_establish_one_authority_and_cannot_reclaim_its_tombstone() { + let Some(database_url) = database_url("fresh_writers_establish_one_authority") else { + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-authority-bootstrap-{}", uuid::Uuid::new_v4()); + let fingerprint_a = spec_content_fingerprint("declaration-a"); + let fingerprint_b = spec_content_fingerprint("declaration-b"); + + let writer_a = store.clone(); + let tenant_a = tenant.clone(); + let fingerprint_a_task = fingerprint_a.clone(); + let writer_a = sqlx::__rt::spawn(async move { + writer_a + .append_with_index_rows( + &format!("{tenant_a}:Item:item-a"), + 0, + &[test_envelope("CreatedByA")], + &[], + &[], + false, + Some(&fingerprint_a_task), + ) + .await + }); + let writer_b = store.clone(); + let tenant_b = tenant.clone(); + let fingerprint_b_task = fingerprint_b.clone(); + let writer_b = sqlx::__rt::spawn(async move { + writer_b + .append_with_index_rows( + &format!("{tenant_b}:Item:item-b"), + 0, + &[test_envelope("CreatedByB")], + &[], + &[], + false, + Some(&fingerprint_b_task), + ) + .await + }); + let results = [writer_a.await, writer_b.await]; + assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1); + assert_eq!( + results + .iter() + .filter(|result| matches!( + result, + Err(PersistenceError::Storage(message)) + if message.contains("stale spec declaration fingerprint") + )) + .count(), + 1 + ); + + let authority: (i64, String, bool) = crate::dbm::postgres_query_as!( + "SELECT revision, declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read authority"); + assert_eq!(authority.0, 1); + assert!(authority.1 == fingerprint_a || authority.1 == fingerprint_b); + assert!(authority.2); + + store + .delete_spec(&tenant, "Item") + .await + .expect("tombstone compatibility authority"); + let stale = store + .append_with_index_rows( + &format!("{tenant}:Item:item-after-delete"), + 0, + &[test_envelope("Created")], + &[], + &[], + false, + Some(&authority.1), + ) + .await + .expect_err("tombstone cannot be reclaimed"); + assert!(matches!( + stale, + PersistenceError::Storage(message) + if message.contains("stale spec declaration fingerprint") + )); + }); +} + +#[test] +fn fresh_reconciliation_bootstraps_revision_one_not_the_caller_revision() { + let Some(database_url) = database_url("fresh_reconciliation_bootstraps_revision_one") else { + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-reconciliation-bootstrap-{}", uuid::Uuid::new_v4()); + let fingerprint = spec_content_fingerprint("fresh-vector-declaration"); + + assert_eq!( + store + .begin_vector_index_reconciliation( + &tenant, + "Item", + "v2|embed", + u64::MAX, + &fingerprint, + ) + .await + .expect("bootstrap reconciliation"), + 1 + ); + let authority: (i64, String, bool) = crate::dbm::postgres_query_as!( + "SELECT revision, declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read authority"); + assert_eq!(authority, (1, fingerprint, true)); + }); +} + +#[test] +fn staged_spec_does_not_advance_authority_until_commit() { + let Some(database_url) = database_url("staged_spec_does_not_advance_authority") else { + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-staged-authority-{}", uuid::Uuid::new_v4()); + let ioa_a = "[automaton]\nname = \"Item\"\n# committed-a\n"; + let ioa_b = "[automaton]\nname = \"Item\"\n# staged-b\n"; + let fingerprint_a = spec_content_fingerprint(ioa_a); + let fingerprint_b = spec_content_fingerprint(ioa_b); + let csdl = ""; + + store + .upsert_spec(&tenant, "Item", ioa_a, csdl, &fingerprint_a) + .await + .expect("stage A"); + store.commit_specs(&tenant).await.expect("commit A"); + let generation_a = store + .begin_vector_index_reconciliation(&tenant, "Item", "v2|a", 1, &fingerprint_a) + .await + .expect("begin A"); + store + .mark_vector_index_backfilled(&tenant, "Item", generation_a, "v2|a") + .await + .expect("publish A watermark"); + let authority_a: (i64, String, bool) = crate::dbm::postgres_query_as!( + "SELECT revision, declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read A authority"); + + store + .upsert_spec(&tenant, "Item", ioa_b, csdl, &fingerprint_b) + .await + .expect("stage B"); + let staged_authority: (i64, String, bool) = crate::dbm::postgres_query_as!( + "SELECT revision, declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read staged authority"); + assert_eq!(staged_authority, authority_a); + assert_eq!( + store + .vector_index_backfilled_types(&tenant) + .await + .expect("A watermark during staging"), + vec![("Item".to_string(), "v2|a".to_string())] + ); + + crate::dbm::postgres_query!( + "DELETE FROM staged_specs WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .execute(store.pool()) + .await + .expect("discard staged B"); + let discarded_authority: (i64, String, bool) = crate::dbm::postgres_query_as!( + "SELECT revision, declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read authority after discard"); + assert_eq!(discarded_authority, authority_a); + + store + .upsert_spec(&tenant, "Item", ioa_b, csdl, &fingerprint_b) + .await + .expect("restage B"); + store.commit_specs(&tenant).await.expect("commit B"); + let authority_b: (i64, String, bool) = crate::dbm::postgres_query_as!( + "SELECT revision, declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read B authority"); + assert!(authority_b.0 > authority_a.0); + assert_eq!(authority_b.1, fingerprint_b); + assert!(authority_b.2); + assert!( + store + .vector_index_backfilled_types(&tenant) + .await + .expect("watermark after B commit") + .is_empty(), + "the false-to-true commit transition must withdraw A's watermark" + ); + }); +} + +#[test] +fn full_replacement_tombstones_authority_hidden_by_staged_catalog_row() { + let Some(database_url) = + database_url("full_replacement_tombstones_authority_hidden_by_staged_catalog_row") + else { + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-staged-replacement-{}", uuid::Uuid::new_v4()); + let ioa_a = "[automaton]\nname = \"Item\"\n# committed-a\n"; + let ioa_b = "[automaton]\nname = \"Item\"\n# staged-b\n"; + let fingerprint_a = spec_content_fingerprint(ioa_a); + let fingerprint_b = spec_content_fingerprint(ioa_b); + let csdl = ""; + + store + .upsert_spec(&tenant, "Item", ioa_a, csdl, &fingerprint_a) + .await + .expect("stage A"); + store.commit_specs(&tenant).await.expect("commit A"); + let generation_a = store + .begin_vector_index_reconciliation(&tenant, "Item", "v2|a", 1, &fingerprint_a) + .await + .expect("begin A"); + store + .mark_vector_index_backfilled(&tenant, "Item", generation_a, "v2|a") + .await + .expect("publish A watermark"); + let authority_a: (i64, String, bool) = crate::dbm::postgres_query_as!( + "SELECT revision, declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read A authority"); + + store + .upsert_spec(&tenant, "Item", ioa_b, csdl, &fingerprint_b) + .await + .expect("stage B"); + let staged_authority: (i64, String, bool) = crate::dbm::postgres_query_as!( + "SELECT revision, declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read staged authority"); + assert_eq!(staged_authority, authority_a); + + assert_eq!( + store + .persist_spec_catalog_update(&tenant, &[], csdl, &[], true, None) + .await + .expect("replace with empty catalog"), + vec!["Item".to_string()] + ); + let tombstone: (i64, String, bool) = crate::dbm::postgres_query_as!( + "SELECT revision, declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read tombstone"); + assert!(tombstone.0 > authority_a.0); + assert_eq!(tombstone.1, "absent:v1"); + assert!(!tombstone.2); + assert!( + store + .vector_index_backfilled_types(&tenant) + .await + .expect("watermarks after omission") + .is_empty(), + "full replacement must withdraw the committed declaration even when its catalog row is staged" + ); + }); +} + +#[test] +fn verified_commit_rejects_same_type_fingerprint_overwrite() { + let Some(database_url) = database_url("verified_commit_rejects_same_type_overwrite") else { + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-same-type-commit-{}", uuid::Uuid::new_v4()); + let ioa_a = "[automaton]\nname = \"Item\"\n# committed-a\n"; + let ioa_b = "[automaton]\nname = \"Item\"\n# staged-b\n"; + let fingerprint_a = spec_content_fingerprint(ioa_a); + let fingerprint_b = spec_content_fingerprint(ioa_b); + let csdl = ""; + + store + .upsert_spec(&tenant, "Item", ioa_a, csdl, &fingerprint_a) + .await + .expect("stage A"); + store + .commit_verified_spec( + &tenant, + "Item", + &fingerprint_a, + csdl, + crate::PostgresSpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .expect("commit verified A"); + store + .upsert_spec(&tenant, "Item", ioa_b, csdl, &fingerprint_b) + .await + .expect("stage B over A"); + + let error = store + .commit_verified_spec( + &tenant, + "Item", + &fingerprint_a, + csdl, + crate::PostgresSpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .expect_err("verified A must not publish staged B"); + assert!(error.to_string().contains("fingerprint changed")); + + let committed_a: (String, bool, bool) = crate::dbm::postgres_query_as!( + "SELECT content_hash, verified, committed FROM specs \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read committed A"); + assert_eq!(committed_a, (fingerprint_a.clone(), true, true)); + let staged_b: (String,) = crate::dbm::postgres_query_as!( + "SELECT content_hash FROM staged_specs \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read staged B"); + assert_eq!(staged_b.0, fingerprint_b); + let authority: (String, bool) = crate::dbm::postgres_query_as!( + "SELECT declaration_fingerprint, present FROM spec_declaration_authority \ + WHERE tenant = $1 AND entity_type = 'Item'", + ) + .bind(&tenant) + .fetch_one(store.pool()) + .await + .expect("read committed A authority"); + assert_eq!(authority, (fingerprint_a, true)); + }); +} + +#[test] +fn verification_cache_ignores_staged_specs_until_commit() { + let Some(database_url) = database_url("verification_cache_ignores_staged_specs_until_commit") + else { + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-staged-cache-{}", uuid::Uuid::new_v4()); + let ioa_source = "[automaton]\nname = \"Issue\"\n"; + let csdl = ""; + let content_hash = spec_content_fingerprint(ioa_source); + + store + .upsert_spec(&tenant, "Issue", ioa_source, csdl, &content_hash) + .await + .expect("stage Issue"); + assert!( + !store + .load_verification_cache(&tenant) + .await + .expect("load staged cache") + .contains_key("Issue"), + "staged verification must not make bootstrap skip durable publication" + ); + + store + .commit_verified_spec( + &tenant, + "Issue", + &content_hash, + csdl, + crate::PostgresSpecVerificationUpdate { + status: "passed", + verified: true, + levels_passed: Some(1), + levels_total: Some(1), + verification_result_json: Some(r#"{"all_passed":true}"#), + }, + ) + .await + .expect("verify and commit Issue"); + assert_eq!( + store + .load_verification_cache(&tenant) + .await + .expect("load committed cache") + .get("Issue"), + Some(&(content_hash, true)) + ); + }); +} + +#[test] +fn existing_authority_writers_share_the_fence_while_spec_mutation_waits() { + let Some(database_url) = database_url("existing_authority_writers_share_the_fence") else { + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-shared-authority-{}", uuid::Uuid::new_v4()); + let ioa_a = "[automaton]\nname = \"Item\"\n# shared-a\n"; + let ioa_b = "[automaton]\nname = \"Item\"\n# shared-b\n"; + let fingerprint_a = spec_content_fingerprint(ioa_a); + let fingerprint_b = spec_content_fingerprint(ioa_b); + let csdl = ""; + store + .upsert_spec(&tenant, "Item", ioa_a, csdl, &fingerprint_a) + .await + .expect("persist authority A"); + store.commit_specs(&tenant).await.expect("commit A"); + + let mut writer_a = store.pool().begin().await.expect("writer A transaction"); + PostgresEventStore::validate_live_spec_declaration( + &mut writer_a, + &tenant, + "Item", + &fingerprint_a, + ) + .await + .expect("writer A shared fence"); + + let mut writer_b = store.pool().begin().await.expect("writer B transaction"); + sqlx::__rt::timeout( + Duration::from_secs(1), + PostgresEventStore::validate_live_spec_declaration( + &mut writer_b, + &tenant, + "Item", + &fingerprint_a, + ), + ) + .await + .expect("existing-authority writer B must not serialize behind writer A") + .expect("writer B shared fence"); + + let mut mutation = { + let mutation_store = store.clone(); + let mutation_tenant = tenant.clone(); + let mutation_fingerprint = fingerprint_b.clone(); + sqlx::__rt::spawn(async move { + mutation_store + .upsert_spec(&mutation_tenant, "Item", ioa_b, csdl, &mutation_fingerprint) + .await?; + mutation_store.commit_specs(&mutation_tenant).await + }) + }; + assert!( + sqlx::__rt::timeout(Duration::from_millis(100), &mut mutation) + .await + .is_err(), + "spec publication must wait for writer A and writer B" + ); + writer_a.commit().await.expect("commit writer A"); + assert!( + sqlx::__rt::timeout(Duration::from_millis(100), &mut mutation) + .await + .is_err(), + "spec publication must still wait for writer B" + ); + writer_b.commit().await.expect("commit writer B"); + mutation + .await + .expect("spec publication after both shared fences"); + }); +} diff --git a/crates/temper-store-postgres/src/store_projection_test.rs b/crates/temper-store-postgres/src/store_projection_test.rs index 22f3e2df1..0932c0785 100644 --- a/crates/temper-store-postgres/src/store_projection_test.rs +++ b/crates/temper-store-postgres/src/store_projection_test.rs @@ -797,6 +797,121 @@ fn native_data_only_create_inserts_event_catalog_and_index_atomically() { }); } +#[test] +fn native_data_only_create_rejects_a_stale_fingerprint_before_any_insert() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + + sqlx::test_block_on(async { + let pool = PgPool::connect(&database_url).await.unwrap(); + run_migrations(&pool).await.unwrap(); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("tenant-native-fingerprint-{}", uuid::Uuid::new_v4()); + let entity_type = "SessionEntry"; + let entity_id = "entry-stale"; + let ioa_a = "[automaton]\nname = \"SessionEntry\"\n# declaration-a\n"; + let ioa_b = "[automaton]\nname = \"SessionEntry\"\n# declaration-b\n"; + let fingerprint_a = spec_content_fingerprint(ioa_a); + let fingerprint_b = spec_content_fingerprint(ioa_b); + let csdl = ""; + store + .upsert_spec(&tenant, entity_type, ioa_a, csdl, &fingerprint_a) + .await + .unwrap(); + store.commit_specs(&tenant).await.unwrap(); + store + .upsert_spec(&tenant, entity_type, ioa_b, csdl, &fingerprint_b) + .await + .unwrap(); + store.commit_specs(&tenant).await.unwrap(); + + let fields = serde_json::json!({"Id": entity_id, "Content": "stale"}); + let state = serde_json::json!({ + "entity_type": entity_type, + "entity_id": entity_id, + "status": "Active", + "fields": fields, + "sequence_nr": 1 + }); + let mut envelope = test_envelope("Created", fields.clone()); + envelope.sequence_nr = 1; + + let rejected = store + .create_data_only_entity_native_with_state( + &tenant, + entity_type, + entity_id, + "Active", + &fields, + &state, + &envelope, + Some(&fingerprint_a), + ) + .await + .expect_err("declaration A must not write after declaration B is authoritative"); + assert!(matches!( + rejected, + PersistenceError::Storage(message) + if message.contains("stale spec declaration fingerprint") + )); + + let event_count: i64 = crate::dbm::postgres_query_scalar!( + "SELECT COUNT(*)::bigint FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&pool) + .await + .unwrap(); + let catalog_count: i64 = crate::dbm::postgres_query_scalar!( + "SELECT COUNT(*)::bigint FROM entity_catalog \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&pool) + .await + .unwrap(); + let index_count: i64 = crate::dbm::postgres_query_scalar!( + "SELECT COUNT(*)::bigint FROM entity_field_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + (event_count, catalog_count, index_count), + (0, 0, 0), + "fingerprint validation must precede journal and projection writes" + ); + + assert_eq!( + store + .create_data_only_entity_native_with_state( + &tenant, + entity_type, + entity_id, + "Active", + &fields, + &state, + &envelope, + Some(&fingerprint_b), + ) + .await + .expect("the authoritative declaration may create the entity"), + 1 + ); + }); +} + #[test] fn upsert_query_projection_advances_sequence_without_rewriting_unchanged_index() { let database_url = match std::env::var("DATABASE_URL") { diff --git a/crates/temper-store-postgres/src/store_vector_reconciliation_test.rs b/crates/temper-store-postgres/src/store_vector_reconciliation_test.rs new file mode 100644 index 000000000..171496246 --- /dev/null +++ b/crates/temper-store-postgres/src/store_vector_reconciliation_test.rs @@ -0,0 +1,304 @@ +use super::*; +use crate::migration::run_migrations; + +fn database_url(test_name: &str) -> Option { + match std::env::var("DATABASE_URL") { + Ok(url) => Some(url), + Err(_) => { + tracing::warn!( + test_name, + "skipping Postgres integration test: DATABASE_URL is not set" + ); + None + } + } +} + +fn envelope(event_type: &str) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: 0, + event_type: event_type.to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: uuid::Uuid::new_v4(), + causation_id: uuid::Uuid::new_v4(), + correlation_id: uuid::Uuid::new_v4(), + timestamp: chrono::Utc::now(), + actor_id: "vector-test".to_string(), + }, + } +} + +fn vector(decl_name: &str, x: f32, y: f32) -> EntityVectorRow { + EntityVectorRow { + decl_name: decl_name.to_string(), + model_tag: "m1".to_string(), + vector: vec![x, y], + } +} + +#[test] +fn reconciliation_generations_fence_stale_rows_deletion_and_readd() { + let Some(database_url) = database_url("reconciliation_generations_fence_stale_rows") else { + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-vector-generation-{}", uuid::Uuid::new_v4()); + let persistence_id = format!("{tenant}:Item:item-1"); + let ioa_a = "[automaton]\nname = \"Item\"\n# vector-a\n"; + let ioa_b = "[automaton]\nname = \"Item\"\n# vector-b\n"; + let fingerprint_a = spec_content_fingerprint(ioa_a); + let fingerprint_b = spec_content_fingerprint(ioa_b); + let csdl = ""; + + store + .upsert_spec(&tenant, "Item", ioa_a, csdl, &fingerprint_a) + .await + .expect("persist A"); + store.commit_specs(&tenant).await.expect("commit A"); + let generation_a = store + .begin_vector_index_reconciliation(&tenant, "Item", "v2|a", 1, &fingerprint_a) + .await + .expect("begin A"); + store + .mark_vector_index_backfilled(&tenant, "Item", generation_a, "v2|a") + .await + .expect("publish A"); + + store + .upsert_spec(&tenant, "Item", ioa_b, csdl, &fingerprint_b) + .await + .expect("persist B"); + store.commit_specs(&tenant).await.expect("commit B"); + let generation_b = store + .begin_vector_index_reconciliation(&tenant, "Item", "v2|b", 2, &fingerprint_b) + .await + .expect("begin B"); + assert!(generation_b > generation_a); + assert!( + store + .vector_index_backfilled_types(&tenant) + .await + .expect("watermarks") + .is_empty(), + "B must withdraw A's completion before rebuilding" + ); + assert!( + store + .begin_vector_index_reconciliation(&tenant, "Item", "v2|a", 99, &fingerprint_a) + .await + .is_err(), + "caller revision cannot let stale A reclaim B" + ); + + store + .append_with_index_rows( + &persistence_id, + 0, + &[envelope("Created")], + &[], + &[vector("b", 0.0, 1.0)], + true, + Some(&fingerprint_b), + ) + .await + .expect("append live B vector"); + store + .backfill_entity_vectors( + &tenant, + "Item", + "item-1", + generation_b, + 0, + &[vector("a", 1.0, 0.0)], + ) + .await + .expect("ignore older replay"); + assert!( + store + .vector_candidates(&tenant, "Item", "a", "m1", 10) + .await + .expect("A candidates") + .is_empty() + ); + + store + .append_with_index_rows( + &persistence_id, + 1, + &[envelope("Deleted")], + &[], + &[], + true, + Some(&fingerprint_b), + ) + .await + .expect("purge live vectors"); + store + .backfill_entity_vectors( + &tenant, + "Item", + "item-1", + generation_b, + 1, + &[vector("b", 0.0, 1.0)], + ) + .await + .expect("ignore resurrection at delete fence"); + assert!( + store + .vector_candidates(&tenant, "Item", "b", "m1", 10) + .await + .expect("B candidates") + .is_empty() + ); + + store.delete_spec(&tenant, "Item").await.expect("delete B"); + let absent_generation = store + .begin_vector_index_reconciliation( + &tenant, + "Item", + "v2|", + 1, + ABSENT_DECLARATION_FINGERPRINT, + ) + .await + .expect("begin absence"); + store + .upsert_spec(&tenant, "Item", ioa_a, csdl, &fingerprint_a) + .await + .expect("re-add A"); + store.commit_specs(&tenant).await.expect("commit re-add"); + let readded_generation = store + .begin_vector_index_reconciliation(&tenant, "Item", "v2|a", 1, &fingerprint_a) + .await + .expect("begin re-added A"); + assert!(readded_generation > absent_generation); + }); +} + +#[test] +fn stale_live_writer_cannot_advance_single_or_batch_journals() { + let Some(database_url) = database_url("stale_live_writer_cannot_advance_journals") else { + return; + }; + + sqlx::__rt::test_block_on(async { + let pool = PgPool::connect(&database_url).await.expect("connect"); + run_migrations(&pool).await.expect("migrate"); + let store = PostgresEventStore::new(pool); + let tenant = format!("tenant-stale-vector-writer-{}", uuid::Uuid::new_v4()); + let item_id = format!("{tenant}:Item:item-1"); + let audit_id = format!("{tenant}:Audit:audit-1"); + let ioa_a = "[automaton]\nname = \"Item\"\n# writer-a\n"; + let ioa_b = "[automaton]\nname = \"Item\"\n# writer-b\n"; + let fingerprint_a = spec_content_fingerprint(ioa_a); + let fingerprint_b = spec_content_fingerprint(ioa_b); + let csdl = ""; + + store + .upsert_spec(&tenant, "Item", ioa_a, csdl, &fingerprint_a) + .await + .expect("persist A"); + store.commit_specs(&tenant).await.expect("commit A"); + store + .append_with_index_rows( + &item_id, + 0, + &[envelope("Created")], + &[], + &[vector("a", 1.0, 0.0)], + true, + Some(&fingerprint_a), + ) + .await + .expect("append A"); + + store + .upsert_spec(&tenant, "Item", ioa_b, csdl, &fingerprint_b) + .await + .expect("persist B"); + store.commit_specs(&tenant).await.expect("commit B"); + let generation_b = store + .begin_vector_index_reconciliation(&tenant, "Item", "v2|b", 2, &fingerprint_b) + .await + .expect("begin B"); + store + .backfill_entity_vectors( + &tenant, + "Item", + "item-1", + generation_b, + 1, + &[vector("b", 0.0, 1.0)], + ) + .await + .expect("install B"); + + let stale_single = store + .append_with_index_rows( + &item_id, + 1, + &[envelope("StaleUpdated")], + &[], + &[], + false, + Some(&fingerprint_a), + ) + .await + .expect_err("stale non-vector write"); + assert!(matches!( + stale_single, + PersistenceError::Storage(message) + if message.contains("stale spec declaration fingerprint") + )); + + let stale_batch = store + .append_batch(&[ + PersistenceAppend { + persistence_id: audit_id.clone(), + expected_sequence: 0, + events: vec![envelope("Recorded")], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, + }, + PersistenceAppend { + persistence_id: item_id.clone(), + expected_sequence: 1, + events: vec![envelope("StaleBatchUpdated")], + vector_rows: vec![vector("a", 1.0, 0.0)], + reconcile_vectors: true, + spec_declaration_fingerprint: Some(fingerprint_a), + }, + ]) + .await + .expect_err("stale batch writer"); + assert!(matches!( + stale_batch, + PersistenceError::Storage(message) + if message.contains("stale spec declaration fingerprint") + )); + assert_eq!(store.read_events(&item_id, 0).await.unwrap().len(), 1); + assert!(store.read_events(&audit_id, 0).await.unwrap().is_empty()); + assert!( + store + .vector_candidates(&tenant, "Item", "a", "m1", 10) + .await + .unwrap() + .is_empty() + ); + assert_eq!( + store + .vector_candidates(&tenant, "Item", "b", "m1", 10) + .await + .unwrap()[0] + .vector, + vec![0.0, 1.0] + ); + }); +} diff --git a/crates/temper-store-sim/src/lib.rs b/crates/temper-store-sim/src/lib.rs index d52a96643..9bd9af67a 100644 --- a/crates/temper-store-sim/src/lib.rs +++ b/crates/temper-store-sim/src/lib.rs @@ -18,6 +18,8 @@ use temper_runtime::persistence::{ }; use temper_runtime::tenant::parse_persistence_id_parts; +const ABSENT_DECLARATION_FINGERPRINT: &str = "absent:v1"; + /// Fault injection configuration for simulation. /// /// Controls the probability of injected failures during event store operations. @@ -28,7 +30,7 @@ pub struct SimFaultConfig { pub write_failure_prob: f64, /// Probability of a spurious concurrency violation on `append()`. pub concurrency_violation_prob: f64, - /// Probability of truncating journal on `read_events()`. + /// Probability of detecting an injected truncated journal read. pub read_truncation_prob: f64, /// Probability of snapshot save failure. pub snapshot_failure_prob: f64, @@ -142,6 +144,10 @@ struct SimEventStoreInner { /// (e.g. proving the key-index backfill treats an unreadable entity as /// `LoadFailed` and does not watermark its type). See `fail_next_reads`. pending_read_failures: BTreeMap, + /// One-shot vector-watermark write failures keyed by `(tenant, entity_type)`. + /// This proves that reconciliation does not advertise completion when its + /// durable convergence claim cannot be persisted. + pending_vector_watermark_failures: BTreeMap<(String, String), usize>, /// One-shot append delays per `persistence_id`. /// /// Used by dispatch retry tests to deterministically model "the actor @@ -165,12 +171,129 @@ struct SimEventStoreInner { /// exact-scan kNN access path. Unlike the key index this has no uniqueness /// constraint; it is derived, rebuildable ranking state. vector_index: BTreeMap<(String, String, String, String, String), Vec>, - /// ADR-0155 backfill watermark: `(tenant, entity_type) -> vector_set` — each - /// completed type mapped to the sorted comma-joined declared vector-path names the - /// backfill covered. Mirrors `key_index_watermark`. + /// ADR-0181 per-entity `(reconciliation_generation, sequence_nr)` fence. + /// Retained even when the entity has no vector rows, so older work cannot + /// overwrite or resurrect them. + vector_index_version: BTreeMap<(String, String, String), (u64, u64)>, + /// ADR-0181 durable `(generation, declaration_revision, fingerprint, + /// vector_set)` authority per type. + vector_reconciliation_generation: BTreeMap<(String, String), (u64, u64, String, String)>, + /// Durable spec-catalog authority independent of vector work. This mirrors + /// the persistent stores' trigger-maintained source/tombstone row without + /// making ordinary non-vector types reconciliation work. + spec_declaration_authority: BTreeMap<(String, String), (u64, String)>, + /// ADR-0155/0181 backfill watermark: `(tenant, entity_type) -> vector_set` — each + /// completed type mapped to the revisioned full-declaration signature the + /// reconciliation covered. Mirrors `key_index_watermark`. vector_index_watermark: BTreeMap<(String, String), String>, } +impl SimEventStoreInner { + fn current_vector_generation(&self, tenant: &str, entity_type: &str) -> u64 { + self.vector_reconciliation_generation + .get(&(tenant.to_string(), entity_type.to_string())) + .map(|(generation, _, _, _)| *generation) + .unwrap_or(0) + } + + fn stage_live_spec_declaration( + &self, + staged_authority: &mut BTreeMap<(String, String), (u64, String)>, + tenant: &str, + entity_type: &str, + reconcile_vectors: bool, + spec_declaration_fingerprint: Option<&str>, + ) -> Result<(), PersistenceError> { + let key = (tenant.to_string(), entity_type.to_string()); + if reconcile_vectors && spec_declaration_fingerprint.is_none() { + return Err(PersistenceError::Storage(format!( + "vector-index write is missing a spec declaration fingerprint for {tenant}:{entity_type}" + ))); + } + let Some(writer_fingerprint) = spec_declaration_fingerprint else { + return Ok(()); + }; + if let Some((_, current_fingerprint)) = self + .spec_declaration_authority + .get(&key) + .or_else(|| staged_authority.get(&key)) + { + if current_fingerprint != writer_fingerprint { + return Err(PersistenceError::Storage(format!( + "stale live vector declaration fingerprint for {tenant}:{entity_type}" + ))); + } + } else { + // Direct actor tests historically had no separate spec catalog. Keep + // that bootstrap capability, but stage it until the append is known + // to commit so a later validation failure cannot leak authority. + staged_authority.insert(key, (1, writer_fingerprint.to_string())); + } + Ok(()) + } + + fn validate_live_vector_fence( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + new_sequence: u64, + ) -> Result { + let generation = self.current_vector_generation(tenant, entity_type); + if self + .vector_index_version + .get(&( + tenant.to_string(), + entity_type.to_string(), + entity_id.to_string(), + )) + .is_some_and(|(fence_generation, fence_sequence)| { + *fence_generation > generation + || (*fence_generation == generation && *fence_sequence > new_sequence) + }) + { + return Err(PersistenceError::Storage(format!( + "vector-index fence for {tenant}:{entity_type}:{entity_id} is ahead of live journal sequence {new_sequence} in reconciliation generation {generation}" + ))); + } + Ok(generation) + } + + fn apply_live_vector_rows( + &mut self, + tenant: &str, + entity_type: &str, + entity_id: &str, + generation: u64, + new_sequence: u64, + vector_rows: &[EntityVectorRow], + ) { + self.vector_index_version.insert( + ( + tenant.to_string(), + entity_type.to_string(), + entity_id.to_string(), + ), + (generation, new_sequence), + ); + self.vector_index.retain(|(t, et, _, _, eid), _| { + !(t == tenant && et == entity_type && eid == entity_id) + }); + for row in vector_rows { + self.vector_index.insert( + ( + tenant.to_string(), + entity_type.to_string(), + row.decl_name.clone(), + row.model_tag.clone(), + entity_id.to_string(), + ), + row.vector.clone(), + ); + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct SimEventSegment { pub segment_index: u64, @@ -194,10 +317,14 @@ impl SimEventStore { faults, pending_concurrency_violations: BTreeMap::new(), pending_read_failures: BTreeMap::new(), + pending_vector_watermark_failures: BTreeMap::new(), pending_append_delays: BTreeMap::new(), key_index: BTreeMap::new(), key_index_watermark: BTreeMap::new(), vector_index: BTreeMap::new(), + vector_index_version: BTreeMap::new(), + vector_reconciliation_generation: BTreeMap::new(), + spec_declaration_authority: BTreeMap::new(), vector_index_watermark: BTreeMap::new(), })), } @@ -205,13 +332,13 @@ impl SimEventStore { /// Inject exactly `count` deterministic `ConcurrencyViolation` errors on /// the next `count` `append` calls for `persistence_id`, then behave - /// normally. + /// normally. Each injected violation reports the exact durable journal + /// sequence without inventing a write that replay cannot observe. /// /// Use this for retry-path tests where the probabilistic fault injection - /// in `SimFaultConfig` would be flaky. Each injected violation reports - /// `actual = expected_sequence` (the journal has not actually moved), so - /// any callers with post-replay sequence assertions still hold after the - /// retry replays back to the same spot. + /// in `SimFaultConfig` would be flaky. The journal is never mutated by the + /// injected failure, and `actual` can differ from the caller's stale + /// `expected_sequence`. pub fn inject_concurrency_violations(&self, persistence_id: &str, count: u64) { let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock if count == 0 { @@ -225,7 +352,7 @@ impl SimEventStore { /// Make the next `count` `read_events` calls for `persistence_id` fail with a /// storage error, then behave normally. Deterministic (unlike - /// `read_truncation_prob`) so tests can prove read-failure handling — e.g. that + /// `read_truncation_prob`) so tests can target read-failure handling — e.g. that /// the key-index backfill classifies an unreadable entity as `LoadFailed` and /// therefore does not watermark its type. `count == 0` clears the injection. pub fn fail_next_reads(&self, persistence_id: &str, count: usize) { @@ -239,6 +366,18 @@ impl SimEventStore { } } + /// Make the next `count` vector-watermark writes fail for a type, then behave + /// normally. `count == 0` clears the deterministic injection. + pub fn fail_next_vector_watermarks(&self, tenant: &str, entity_type: &str, count: usize) { + let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + let key = (tenant.to_string(), entity_type.to_string()); + if count == 0 { + inner.pending_vector_watermark_failures.remove(&key); + } else { + inner.pending_vector_watermark_failures.insert(key, count); + } + } + /// Return the current count of pending injected concurrency violations for /// `persistence_id`. Zero if none are queued. pub fn pending_concurrency_violations(&self, persistence_id: &str) -> u64 { @@ -271,6 +410,53 @@ impl SimEventStore { Self::new(seed, SimFaultConfig::none()) } + /// Commit a simulated spec source/tombstone before publishing it to a + /// rebuilt registry. + /// + /// The persistent stores do this with triggers on `specs`; this explicit + /// deterministic hook gives restart/failover tests the same durable ordering + /// point without coupling the event store to a platform metadata store. + pub fn persist_spec_declaration( + &self, + tenant: &str, + entity_type: &str, + declaration_fingerprint: &str, + ) -> u64 { + assert!(!declaration_fingerprint.is_empty()); + let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + let key = (tenant.to_string(), entity_type.to_string()); + if let Some((revision, current_fingerprint)) = inner.spec_declaration_authority.get(&key) + && current_fingerprint == declaration_fingerprint + { + return *revision; + } + let next_revision = inner + .spec_declaration_authority + .get(&key) + .map(|(revision, _)| { + revision + .checked_add(1) + .expect("sim spec declaration revision exhausted") + }) + .unwrap_or(1); + inner.spec_declaration_authority.insert( + key.clone(), + (next_revision, declaration_fingerprint.to_string()), + ); + if let Some((generation, revision, fingerprint, vector_set)) = + inner.vector_reconciliation_generation.get_mut(&key) + { + *generation = generation + .checked_add(1) + .expect("sim vector reconciliation generation exhausted"); + *revision = next_revision; + fingerprint.clear(); + vector_set.clear(); + } + inner.vector_index_watermark.remove(&key); + next_revision + } + /// Return the total number of events across all journals. pub fn total_events(&self) -> usize { let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock @@ -354,8 +540,16 @@ impl EventStore for SimEventStore { expected_sequence: u64, events: &[PersistenceEnvelope], ) -> Result { - self.append_with_index_rows(persistence_id, expected_sequence, events, &[], &[], false) - .await + self.append_with_index_rows( + persistence_id, + expected_sequence, + events, + &[], + &[], + false, + None, + ) + .await } async fn append_with_index_rows( @@ -366,6 +560,7 @@ impl EventStore for SimEventStore { key_rows: &[temper_runtime::persistence::EntityKeyRow], vector_rows: &[EntityVectorRow], reconcile_vectors: bool, + spec_declaration_fingerprint: Option<&str>, ) -> Result { let append_delay = { let mut inner = self @@ -392,15 +587,18 @@ impl EventStore for SimEventStore { } let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + let current_seq = inner + .journals + .get(persistence_id) + .and_then(|journal| journal.last().map(|event| event.sequence_nr)) + .unwrap_or(0); // Deterministic one-shot injection (see `inject_concurrency_violations`). // Consumes one counter per call; falls back to normal flow once drained. // - // The reported `actual` equals `expected_sequence` — the journal has - // not actually moved, so an authoritative replay will land back at - // `expected_sequence`. Any code that asserts - // `post_replay_sequence >= actual` still holds without this injection - // lying about journal state. + // The reported `actual` is always the durable journal sequence. Fault + // injection rejects this append but never invents a write that replay + // cannot observe. let pending_cv = inner .pending_concurrency_violations .get(persistence_id) @@ -416,16 +614,17 @@ impl EventStore for SimEventStore { } return Err(PersistenceError::ConcurrencyViolation { expected: expected_sequence, - actual: expected_sequence, + actual: current_seq, }); } - // Fault injection: spurious concurrency violation (probabilistic). + // Fault injection: spurious concurrency violation (probabilistic). The + // rejection is spurious; its authoritative sequence still is not. let cv_prob = inner.faults.concurrency_violation_prob; if inner.rng.chance(cv_prob) { return Err(PersistenceError::ConcurrencyViolation { expected: expected_sequence, - actual: expected_sequence.wrapping_add(1), + actual: current_seq, }); } @@ -438,11 +637,6 @@ impl EventStore for SimEventStore { } // Check optimistic concurrency. - let current_seq = inner - .journals - .get(persistence_id) - .and_then(|journal| journal.last().map(|e| e.sequence_nr)) - .unwrap_or(0); if current_seq != expected_sequence { return Err(PersistenceError::ConcurrencyViolation { expected: expected_sequence, @@ -450,6 +644,42 @@ impl EventStore for SimEventStore { }); } + // Match the durable stores' live-write invariant: a repair is never + // allowed to claim a journal sequence that the stream has not reached. + // Validate before mutating the journal so a violated fence is atomic. + let mut staged_spec_authority = BTreeMap::new(); + let live_vector_context = if reconcile_vectors || spec_declaration_fingerprint.is_some() { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + inner.stage_live_spec_declaration( + &mut staged_spec_authority, + tenant, + entity_type, + reconcile_vectors, + spec_declaration_fingerprint, + )?; + let new_sequence = expected_sequence + events.len() as u64; + if reconcile_vectors { + let generation = inner.validate_live_vector_fence( + tenant, + entity_type, + entity_id, + new_sequence, + )?; + Some(( + tenant.to_string(), + entity_type.to_string(), + entity_id.to_string(), + generation, + new_sequence, + )) + } else { + None + } + } else { + None + }; + // ADR-0153: validate declared-key uniqueness BEFORE writing the journal, so // a reject is atomic — the journal must not advance on a rejected co-commit. // A *different* entity already holding the key is the violation. @@ -474,6 +704,12 @@ impl EventStore for SimEventStore { } } + // No validation below this point can fail. Publish any compatibility + // bootstrap under the same lock as the journal and derived rows. + inner + .spec_declaration_authority + .extend(staged_spec_authority); + let mut new_seq = expected_sequence; let mut stored_events = Vec::with_capacity(events.len()); for event in events { @@ -560,26 +796,18 @@ impl EventStore for SimEventStore { // the current ones — so a delete transition or a cleared vector/model // property (empty `vector_rows`) purges the stale rows instead of leaving // them to rank forever. No uniqueness constraint — vectors are derived state. - if reconcile_vectors { - let mut parts = persistence_id.splitn(3, ':'); - let tenant = parts.next().unwrap_or(""); - let entity_type = parts.next().unwrap_or(""); - let entity_id = parts.next().unwrap_or(""); - inner.vector_index.retain(|(t, et, _, _, eid), _| { - !(t.as_str() == tenant && et.as_str() == entity_type && eid.as_str() == entity_id) - }); - for row in vector_rows { - inner.vector_index.insert( - ( - tenant.to_string(), - entity_type.to_string(), - row.decl_name.clone(), - row.model_tag.clone(), - entity_id.to_string(), - ), - row.vector.clone(), - ); - } + if let Some((tenant, entity_type, entity_id, generation, expected_new_sequence)) = + live_vector_context + { + debug_assert_eq!(expected_new_sequence, new_seq); + inner.apply_live_vector_rows( + &tenant, + &entity_type, + &entity_id, + generation, + new_seq, + vector_rows, + ); } Ok(new_seq) @@ -679,14 +907,171 @@ impl EventStore for SimEventStore { Ok(ids.into_iter().collect()) } + async fn persist_spec_declaration( + &self, + tenant: &str, + entity_type: &str, + declaration_fingerprint: &str, + ) -> Result { + Ok(SimEventStore::persist_spec_declaration( + self, + tenant, + entity_type, + declaration_fingerprint, + )) + } + + async fn spec_declaration_entity_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + Ok(inner + .spec_declaration_authority + .iter() + .filter(|((stored_tenant, _), (_, fingerprint))| { + stored_tenant == tenant && fingerprint != ABSENT_DECLARATION_FINGERPRINT + }) + .map(|((_, entity_type), _)| entity_type.clone()) + .collect()) + } + + async fn begin_vector_index_reconciliation( + &self, + tenant: &str, + entity_type: &str, + vector_set: &str, + declaration_revision: u64, + declaration_fingerprint: &str, + ) -> Result { + if declaration_revision == 0 || declaration_fingerprint.is_empty() { + return Err(PersistenceError::Storage(format!( + "vector declaration revision must be nonzero and fingerprinted for {tenant}:{entity_type}" + ))); + } + let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + let key = (tenant.to_string(), entity_type.to_string()); + let authoritative_revision = match inner.spec_declaration_authority.get(&key).cloned() { + Some((revision, fingerprint)) if fingerprint == declaration_fingerprint => revision, + Some((revision, _)) => { + return Err(PersistenceError::Storage(format!( + "vector declaration fingerprint does not match durable authority for {tenant}:{entity_type} at revision {revision}; caller-local revision {declaration_revision} cannot supersede it" + ))); + } + None => { + // Compatibility bootstrap for direct EventStore users without a + // simulated catalog. Once present, only persist_spec_declaration + // may change this authority. + inner + .spec_declaration_authority + .insert(key.clone(), (1, declaration_fingerprint.to_string())); + 1 + } + }; + let current = inner.vector_reconciliation_generation.get(&key).cloned(); + let Some((generation, current_revision, current_fingerprint, current_set)) = current else { + inner.vector_reconciliation_generation.insert( + key.clone(), + ( + 1, + authoritative_revision, + declaration_fingerprint.to_string(), + vector_set.to_string(), + ), + ); + inner.vector_index_watermark.remove(&key); + return Ok(1); + }; + // A rebuilt ServerState may restart its process-local registry revision at + // one. The durable fingerprint/set is the idempotency identity: an exact + // restart resumes the existing generation even when its local revision is + // lower than the stored diagnostic revision. + if generation > 0 + && current_fingerprint == declaration_fingerprint + && current_set == vector_set + { + return Ok(generation); + } + if authoritative_revision < current_revision { + return Err(PersistenceError::Storage(format!( + "vector reconciliation revision {current_revision} exceeds declaration authority {authoritative_revision} for {tenant}:{entity_type}" + ))); + } + if authoritative_revision == current_revision + && generation > 0 + && (!current_fingerprint.is_empty() || !current_set.is_empty()) + { + return Err(PersistenceError::Storage(format!( + "conflicting vector declaration at revision {authoritative_revision} for {tenant}:{entity_type}" + ))); + } + let next_generation = if authoritative_revision == current_revision { + generation.max(1) + } else { + generation.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "vector reconciliation generation exhausted for {tenant}:{entity_type}" + )) + })? + }; + inner.vector_reconciliation_generation.insert( + key.clone(), + ( + next_generation, + authoritative_revision, + declaration_fingerprint.to_string(), + vector_set.to_string(), + ), + ); + // A new generation makes the previous completion signature non-authoritative. + // Remove it under the same lock as the generation advance so another + // coordinator cannot observe the old signature and incorrectly skip. + inner.vector_index_watermark.remove(&key); + Ok(next_generation) + } + async fn backfill_entity_vectors( &self, tenant: &str, entity_type: &str, entity_id: &str, + reconciliation_generation: u64, + observed_sequence: u64, vector_rows: &[EntityVectorRow], ) -> Result<(), PersistenceError> { + if reconciliation_generation == 0 { + return Err(PersistenceError::Storage( + "vector reconciliation generation zero is reserved for pre-reconciliation live writes" + .to_string(), + )); + } let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + let current_generation = inner.current_vector_generation(tenant, entity_type); + if current_generation != reconciliation_generation { + return Err(PersistenceError::Storage(format!( + "stale vector reconciliation generation {reconciliation_generation} for {tenant}:{entity_type}; current generation is {current_generation}" + ))); + } + let version_key = ( + tenant.to_string(), + entity_type.to_string(), + entity_id.to_string(), + ); + if let Some((fence_generation, fence_sequence)) = + inner.vector_index_version.get(&version_key).copied() + { + if fence_generation > reconciliation_generation { + return Err(PersistenceError::Storage(format!( + "vector-index fence generation {fence_generation} is ahead of current type generation {reconciliation_generation} for {tenant}:{entity_type}:{entity_id}" + ))); + } + if fence_generation == reconciliation_generation && fence_sequence > observed_sequence { + return Ok(()); + } + } + inner + .vector_index_version + .insert(version_key, (reconciliation_generation, observed_sequence)); // Reconcile: drop ALL of the entity's rows, then insert the current ones. // Empty `vector_rows` purges the entity (deleted / un-embedded). Idempotent. inner.vector_index.retain(|(t, et, _, _, eid), _| { @@ -742,13 +1127,48 @@ impl EventStore for SimEventStore { &self, tenant: &str, entity_type: &str, + reconciliation_generation: u64, vector_set: &str, ) -> Result<(), PersistenceError> { + if reconciliation_generation == 0 { + return Err(PersistenceError::Storage( + "vector reconciliation generation zero cannot publish a watermark".to_string(), + )); + } let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock - inner.vector_index_watermark.insert( - (tenant.to_string(), entity_type.to_string()), - vector_set.to_string(), - ); + let key = (tenant.to_string(), entity_type.to_string()); + let current = inner.vector_reconciliation_generation.get(&key); + if current.map(|(generation, _, _, signature)| { + *generation == reconciliation_generation && signature == vector_set + }) != Some(true) + { + let current_generation = current + .map(|(generation, _, _, _)| *generation) + .unwrap_or(0); + return Err(PersistenceError::Storage(format!( + "stale vector reconciliation generation {reconciliation_generation} for {tenant}:{entity_type}; current generation is {current_generation}" + ))); + } + let pending = inner + .pending_vector_watermark_failures + .get(&key) + .copied() + .unwrap_or(0); + if pending > 0 { + if pending == 1 { + inner.pending_vector_watermark_failures.remove(&key); + } else { + inner + .pending_vector_watermark_failures + .insert(key, pending - 1); + } + return Err(PersistenceError::Storage( + "SimEventStore: injected vector watermark failure".to_string(), + )); + } + inner + .vector_index_watermark + .insert(key, vector_set.to_string()); Ok(()) } @@ -765,6 +1185,30 @@ impl EventStore for SimEventStore { .collect()) } + async fn vector_reconciliation_entity_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + let mut types = BTreeSet::new(); + for (stored_tenant, entity_type) in inner.vector_reconciliation_generation.keys() { + if stored_tenant == tenant { + types.insert(entity_type.clone()); + } + } + for (stored_tenant, entity_type, _) in inner.vector_index_version.keys() { + if stored_tenant == tenant { + types.insert(entity_type.clone()); + } + } + for (stored_tenant, entity_type, _, _, _) in inner.vector_index.keys() { + if stored_tenant == tenant { + types.insert(entity_type.clone()); + } + } + Ok(types.into_iter().collect()) + } + async fn vectored_entity_ids_for_type( &self, tenant: &str, @@ -800,7 +1244,32 @@ impl EventStore for SimEventStore { } } - for append in appends { + let current_sequences = appends + .iter() + .map(|append| { + inner + .journals + .get(&append.persistence_id) + .and_then(|journal| journal.last()) + .map(|event| event.sequence_nr) + .unwrap_or(0) + }) + .collect::>(); + + // Validate all optimistic-concurrency expectations before consuming + // injected faults. A stale batch must report the exact durable stream + // position and leave both the journals and deterministic fault budget + // untouched. + for (append, current_seq) in appends.iter().zip(¤t_sequences) { + if *current_seq != append.expected_sequence { + return Err(PersistenceError::ConcurrencyViolation { + expected: append.expected_sequence, + actual: *current_seq, + }); + } + } + + for (append, current_seq) in appends.iter().zip(¤t_sequences) { let pending_cv = inner .pending_concurrency_violations .get(&append.persistence_id) @@ -818,7 +1287,7 @@ impl EventStore for SimEventStore { } return Err(PersistenceError::ConcurrencyViolation { expected: append.expected_sequence, - actual: append.expected_sequence, + actual: *current_seq, }); } } @@ -830,7 +1299,7 @@ impl EventStore for SimEventStore { let first = &appends[0]; return Err(PersistenceError::ConcurrencyViolation { expected: first.expected_sequence, - actual: first.expected_sequence.wrapping_add(1), + actual: current_sequences[0], }); } let wf_prob = inner.faults.write_failure_prob; @@ -840,33 +1309,75 @@ impl EventStore for SimEventStore { )); } + // Validate every vector fence before mutating any journal. The later row + // replacement is infallible under this same lock, so journal/fence/candidates + // remain one atomic simulation step. + let mut vector_contexts = Vec::with_capacity(appends.len()); + let mut staged_spec_authority = BTreeMap::new(); for append in appends { - let current_seq = inner - .journals - .get(&append.persistence_id) - .and_then(|journal| journal.last()) - .map(|event| event.sequence_nr) - .unwrap_or(0); - if current_seq != append.expected_sequence { - return Err(PersistenceError::ConcurrencyViolation { - expected: append.expected_sequence, - actual: current_seq, - }); + if append.reconcile_vectors || append.spec_declaration_fingerprint.is_some() { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(&append.persistence_id) + .map_err(PersistenceError::Storage)?; + inner.stage_live_spec_declaration( + &mut staged_spec_authority, + tenant, + entity_type, + append.reconcile_vectors, + append.spec_declaration_fingerprint.as_deref(), + )?; + let new_sequence = append.expected_sequence + append.events.len() as u64; + if append.reconcile_vectors { + let generation = inner.validate_live_vector_fence( + tenant, + entity_type, + entity_id, + new_sequence, + )?; + vector_contexts.push(Some(( + tenant.to_string(), + entity_type.to_string(), + entity_id.to_string(), + generation, + new_sequence, + ))); + } else { + vector_contexts.push(None); + } + } else { + vector_contexts.push(None); } } + inner + .spec_declaration_authority + .extend(staged_spec_authority); let mut results = Vec::with_capacity(appends.len()); - for append in appends { - let journal = inner - .journals - .entry(append.persistence_id.clone()) - .or_default(); + for (append, vector_context) in appends.iter().zip(vector_contexts) { let mut new_seq = append.expected_sequence; - for event in &append.events { - new_seq += 1; - let mut stored = event.clone(); - stored.sequence_nr = new_seq; - journal.push(stored); + { + let journal = inner + .journals + .entry(append.persistence_id.clone()) + .or_default(); + for event in &append.events { + new_seq += 1; + let mut stored = event.clone(); + stored.sequence_nr = new_seq; + journal.push(stored); + } + } + if let Some((tenant, entity_type, entity_id, generation, new_sequence)) = vector_context + { + debug_assert_eq!(new_sequence, new_seq); + inner.apply_live_vector_rows( + &tenant, + &entity_type, + &entity_id, + generation, + new_sequence, + &append.vector_rows, + ); } results.push(PersistenceAppendResult { persistence_id: append.persistence_id.clone(), @@ -901,17 +1412,23 @@ impl EventStore for SimEventStore { None => return Ok(Vec::new()), }; - let mut events: Vec = journal + let events: Vec = journal .iter() .filter(|e| e.sequence_nr > from_sequence) .cloned() .collect(); - // Fault injection: truncate the returned events. + // A caller cannot distinguish a truncated successful prefix from a complete + // journal read. Surface the modeled truncation as corruption instead of + // allowing strict reconciliation to publish state rebuilt from a prefix. let rt_prob = inner.faults.read_truncation_prob; if !events.is_empty() && inner.rng.chance(rt_prob) { let truncate_at = (inner.rng.next_u64() as usize) % events.len(); - events.truncate(truncate_at.max(1)); + return Err(PersistenceError::Storage(format!( + "injected truncated read for {persistence_id}: {}/{} events", + truncate_at.max(1), + events.len() + ))); } Ok(events) diff --git a/crates/temper-store-sim/src/tests.rs b/crates/temper-store-sim/src/tests.rs deleted file mode 100644 index 9e076ff0e..000000000 --- a/crates/temper-store-sim/src/tests.rs +++ /dev/null @@ -1,294 +0,0 @@ -use super::*; -use temper_runtime::persistence::EventMetadata; - -fn test_envelope(seq: u64, event_type: &str) -> PersistenceEnvelope { - PersistenceEnvelope { - sequence_nr: seq, - event_type: event_type.to_string(), - payload: serde_json::json!({"test": true}), - metadata: EventMetadata { - event_id: uuid::Uuid::nil(), - causation_id: uuid::Uuid::nil(), - correlation_id: uuid::Uuid::nil(), - timestamp: chrono::DateTime::UNIX_EPOCH, - actor_id: "test".to_string(), - }, - } -} - -#[tokio::test] -async fn append_and_read_roundtrip() { - let store = SimEventStore::no_faults(42); - let pid = "default:Order:ord-1"; - - let new_seq = store - .append(pid, 0, &[test_envelope(0, "Created")]) - .await - .unwrap(); - assert_eq!(new_seq, 1); - - let events = store.read_events(pid, 0).await.unwrap(); - assert_eq!(events.len(), 1); - assert_eq!(events[0].sequence_nr, 1); - assert_eq!(events[0].event_type, "Created"); -} - -#[tokio::test] -async fn append_multiple_events() { - let store = SimEventStore::no_faults(42); - let pid = "default:Order:ord-2"; - - let seq = store - .append( - pid, - 0, - &[test_envelope(0, "Created"), test_envelope(0, "Submitted")], - ) - .await - .unwrap(); - assert_eq!(seq, 2); - - let events = store.read_events(pid, 0).await.unwrap(); - assert_eq!(events.len(), 2); - assert_eq!(events[0].sequence_nr, 1); - assert_eq!(events[1].sequence_nr, 2); -} - -#[tokio::test] -async fn append_batch_commits_multiple_journals_atomically() { - let store = SimEventStore::no_faults(42); - let appends = vec![ - PersistenceAppend { - persistence_id: "default:Order:ord-a".to_string(), - expected_sequence: 0, - events: vec![test_envelope(0, "Created")], - }, - PersistenceAppend { - persistence_id: "default:Order:ord-b".to_string(), - expected_sequence: 0, - events: vec![test_envelope(0, "Created"), test_envelope(0, "Submitted")], - }, - ]; - - let results = store.append_batch(&appends).await.unwrap(); - - assert_eq!( - results, - vec![ - PersistenceAppendResult { - persistence_id: "default:Order:ord-a".to_string(), - sequence_nr: 1, - }, - PersistenceAppendResult { - persistence_id: "default:Order:ord-b".to_string(), - sequence_nr: 2, - }, - ] - ); - assert_eq!(store.dump_journal("default:Order:ord-a").len(), 1); - assert_eq!(store.dump_journal("default:Order:ord-b").len(), 2); -} - -#[tokio::test] -async fn append_batch_conflict_leaves_all_journals_untouched() { - let store = SimEventStore::no_faults(42); - store - .append( - "default:Order:ord-existing", - 0, - &[test_envelope(0, "Created")], - ) - .await - .unwrap(); - - let err = store - .append_batch(&[ - PersistenceAppend { - persistence_id: "default:Order:ord-new".to_string(), - expected_sequence: 0, - events: vec![test_envelope(0, "Created")], - }, - PersistenceAppend { - persistence_id: "default:Order:ord-existing".to_string(), - expected_sequence: 0, - events: vec![test_envelope(0, "Submitted")], - }, - ]) - .await - .expect_err("second journal conflict should abort entire batch"); - - assert!( - matches!(err, PersistenceError::ConcurrencyViolation { .. }), - "unexpected error: {err}" - ); - assert!( - store.dump_journal("default:Order:ord-new").is_empty(), - "first append must not be persisted when a later stream conflicts" - ); - assert_eq!( - store.dump_journal("default:Order:ord-existing").len(), - 1, - "conflicting stream must keep its original journal only" - ); -} - -#[tokio::test] -async fn concurrency_violation_on_wrong_sequence() { - let store = SimEventStore::no_faults(42); - let pid = "default:Order:ord-3"; - - store - .append(pid, 0, &[test_envelope(0, "Created")]) - .await - .unwrap(); - - let err = store - .append(pid, 0, &[test_envelope(0, "Duplicate")]) - .await - .unwrap_err(); - - assert!(matches!( - err, - PersistenceError::ConcurrencyViolation { - expected: 0, - actual: 1 - } - )); -} - -#[tokio::test] -async fn snapshot_save_and_load() { - let store = SimEventStore::no_faults(42); - let pid = "default:Order:ord-4"; - - store.save_snapshot(pid, 5, b"state-data").await.unwrap(); - - let snap = store.load_snapshot(pid).await.unwrap(); - assert_eq!(snap, Some((5, b"state-data".to_vec()))); -} - -#[tokio::test] -async fn snapshot_save_records_history_and_rotates_segments() { - let store = SimEventStore::no_faults(42); - let pid = "default:Order:segmented"; - - store - .append( - pid, - 0, - &[test_envelope(0, "Created"), test_envelope(0, "Updated")], - ) - .await - .unwrap(); - store.save_snapshot(pid, 2, b"snapshot-2").await.unwrap(); - store - .append(pid, 2, &[test_envelope(0, "AfterSnapshot")]) - .await - .unwrap(); - - assert_eq!(store.snapshot_history_len(pid), 1); - let segments = store.dump_segments(pid); - assert_eq!(segments.len(), 2); - assert_eq!(segments[0].segment_index, 0); - assert_eq!(segments[0].snapshot_sequence, Some(2)); - assert!(segments[0].sealed); - assert_eq!(segments[1].segment_index, 1); - assert_eq!(segments[1].start_sequence_nr, 3); - assert_eq!(segments[1].end_sequence_nr, Some(3)); - assert!(!segments[1].sealed); -} - -#[tokio::test] -async fn load_snapshot_returns_none_when_empty() { - let store = SimEventStore::no_faults(42); - let snap = store - .load_snapshot("default:Order:nonexistent") - .await - .unwrap(); - assert_eq!(snap, None); -} - -#[tokio::test] -async fn list_entity_ids_filters_by_tenant() { - let store = SimEventStore::no_faults(42); - - store - .append("alpha:Order:ord-1", 0, &[test_envelope(0, "Created")]) - .await - .unwrap(); - store - .append("alpha:Task:task-1", 0, &[test_envelope(0, "Created")]) - .await - .unwrap(); - store - .append("beta:Order:ord-9", 0, &[test_envelope(0, "Created")]) - .await - .unwrap(); - - let mut alpha = store.list_entity_ids("alpha").await.unwrap(); - alpha.sort(); - assert_eq!( - alpha, - vec![ - ("Order".to_string(), "ord-1".to_string()), - ("Task".to_string(), "task-1".to_string()), - ] - ); - - let beta = store.list_entity_ids("beta").await.unwrap(); - assert_eq!(beta, vec![("Order".to_string(), "ord-9".to_string())]); -} - -#[tokio::test] -async fn read_events_from_sequence() { - let store = SimEventStore::no_faults(42); - let pid = "default:Order:ord-5"; - - store - .append(pid, 0, &[test_envelope(0, "A"), test_envelope(0, "B")]) - .await - .unwrap(); - store - .append(pid, 2, &[test_envelope(0, "C")]) - .await - .unwrap(); - - // Read from sequence 1 — should skip event at seq 1 - let events = store.read_events(pid, 1).await.unwrap(); - assert_eq!(events.len(), 2); - assert_eq!(events[0].sequence_nr, 2); - assert_eq!(events[1].sequence_nr, 3); -} - -#[tokio::test] -async fn deterministic_across_seeds() { - // Same seed → same behavior (with no faults, behavior is trivially the same) - for seed in [42, 123, 999] { - let store = SimEventStore::no_faults(seed); - let pid = "default:Order:det-1"; - - let seq = store - .append(pid, 0, &[test_envelope(0, "Created")]) - .await - .unwrap(); - assert_eq!(seq, 1); - - let events = store.read_events(pid, 0).await.unwrap(); - assert_eq!(events.len(), 1); - } -} - -#[tokio::test] -async fn fault_injection_produces_errors() { - let faults = SimFaultConfig { - write_failure_prob: 1.0, // always fail - concurrency_violation_prob: 0.0, - read_truncation_prob: 0.0, - snapshot_failure_prob: 0.0, - }; - let store = SimEventStore::new(42, faults); - let pid = "default:Order:fault-1"; - - let err = store.append(pid, 0, &[test_envelope(0, "Created")]).await; - assert!(err.is_err()); -} diff --git a/crates/temper-store-sim/src/tests/mod.rs b/crates/temper-store-sim/src/tests/mod.rs new file mode 100644 index 000000000..270682b58 --- /dev/null +++ b/crates/temper-store-sim/src/tests/mod.rs @@ -0,0 +1,1120 @@ +use super::*; +use temper_runtime::persistence::EventMetadata; + +fn test_envelope(seq: u64, event_type: &str) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: seq, + event_type: event_type.to_string(), + payload: serde_json::json!({"test": true}), + metadata: EventMetadata { + event_id: uuid::Uuid::nil(), + causation_id: uuid::Uuid::nil(), + correlation_id: uuid::Uuid::nil(), + timestamp: chrono::DateTime::UNIX_EPOCH, + actor_id: "test".to_string(), + }, + } +} + +#[tokio::test] +async fn append_and_read_roundtrip() { + let store = SimEventStore::no_faults(42); + let pid = "default:Order:ord-1"; + + let new_seq = store + .append(pid, 0, &[test_envelope(0, "Created")]) + .await + .unwrap(); + assert_eq!(new_seq, 1); + + let events = store.read_events(pid, 0).await.unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].sequence_nr, 1); + assert_eq!(events[0].event_type, "Created"); +} + +#[tokio::test] +async fn append_multiple_events() { + let store = SimEventStore::no_faults(42); + let pid = "default:Order:ord-2"; + + let seq = store + .append( + pid, + 0, + &[test_envelope(0, "Created"), test_envelope(0, "Submitted")], + ) + .await + .unwrap(); + assert_eq!(seq, 2); + + let events = store.read_events(pid, 0).await.unwrap(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].sequence_nr, 1); + assert_eq!(events[1].sequence_nr, 2); +} + +#[tokio::test] +async fn pre_reconciliation_live_vector_type_remains_discoverable() { + let store = SimEventStore::no_faults(41); + store.persist_spec_declaration("default", "Item", "rev-pre"); + store + .append_with_index_rows( + "default:Item:item-before-generation", + 0, + &[test_envelope(0, "Created")], + &[], + &[EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }], + true, + Some("rev-pre"), + ) + .await + .unwrap(); + + assert_eq!( + store + .vector_reconciliation_entity_types("default") + .await + .unwrap(), + vec!["Item".to_string()], + "generation-zero fences must keep remove-all reconciliation discoverable" + ); +} + +#[tokio::test] +async fn stale_vector_backfill_does_not_overwrite_newer_live_write() { + let store = SimEventStore::no_faults(42); + store.persist_spec_declaration("default", "Item", "rev-1"); + let generation = store + .begin_vector_index_reconciliation("default", "Item", "v2|embed", 1, "rev-1") + .await + .unwrap(); + let persistence_id = "default:Item:item-race"; + let stale_row = EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "model-v1".to_string(), + vector: vec![1.0, 0.0], + }; + + store + .append_with_index_rows( + persistence_id, + 0, + &[test_envelope(0, "Created")], + &[], + std::slice::from_ref(&stale_row), + true, + Some("rev-1"), + ) + .await + .unwrap(); + + let live_row = EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "model-v1".to_string(), + vector: vec![0.0, 1.0], + }; + store + .append_with_index_rows( + persistence_id, + 1, + &[test_envelope(0, "Updated")], + &[], + std::slice::from_ref(&live_row), + true, + Some("rev-1"), + ) + .await + .unwrap(); + + // Model a rebuild that loaded journal sequence 1 before the live sequence-2 + // append committed, then reached the index after that append. + store + .backfill_entity_vectors("default", "Item", "item-race", generation, 1, &[stale_row]) + .await + .unwrap(); + + let candidates = store + .vector_candidates("default", "Item", "embed", "model-v1", 10) + .await + .unwrap(); + assert_eq!( + candidates, + vec![EntityVectorCandidate { + entity_id: "item-race".to_string(), + vector: live_row.vector.clone(), + }], + "a stale rebuild observed at sequence 1 must not overwrite the vector co-committed at sequence 2" + ); + + store + .append_with_index_rows( + persistence_id, + 2, + &[test_envelope(0, "Deleted")], + &[], + &[], + true, + Some("rev-1"), + ) + .await + .unwrap(); + store + .backfill_entity_vectors("default", "Item", "item-race", generation, 2, &[live_row]) + .await + .unwrap(); + assert!( + store + .vector_candidates("default", "Item", "embed", "model-v1", 10) + .await + .unwrap() + .is_empty(), + "a stale sequence-2 rebuild must not resurrect vectors purged at sequence 3" + ); + + // Equal-sequence replay is accepted and remains idempotent, including an + // empty tombstone that has no physical vector row. + store + .backfill_entity_vectors("default", "Item", "item-race", generation, 3, &[]) + .await + .unwrap(); + store + .backfill_entity_vectors("default", "Item", "item-race", generation, 3, &[]) + .await + .unwrap(); +} + +#[tokio::test] +async fn newer_vector_reconciliation_generation_rejects_delayed_older_set() { + let store = SimEventStore::no_faults(43); + let persistence_id = "default:Item:item-generation"; + let old_row = EntityVectorRow { + decl_name: "old-embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }; + let new_row = EntityVectorRow { + decl_name: "new-embed".to_string(), + model_tag: "m2".to_string(), + vector: vec![0.0, 1.0], + }; + + store.persist_spec_declaration("default", "Item", "rev-old"); + let old_generation = store + .begin_vector_index_reconciliation("default", "Item", "v2|old-embed", 1, "rev-old") + .await + .unwrap(); + store + .append_with_index_rows( + persistence_id, + 0, + &[test_envelope(0, "Created")], + &[], + std::slice::from_ref(&old_row), + true, + Some("rev-old"), + ) + .await + .unwrap(); + + // The newer declaration set starts and converges from the same journal + // sequence before delayed work from the older invocation resumes. + store.persist_spec_declaration("default", "Item", "rev-new"); + let new_generation = store + .begin_vector_index_reconciliation("default", "Item", "v2|new-embed", 2, "rev-new") + .await + .unwrap(); + store + .backfill_entity_vectors( + "default", + "Item", + "item-generation", + new_generation, + 1, + std::slice::from_ref(&new_row), + ) + .await + .unwrap(); + store + .mark_vector_index_backfilled("default", "Item", new_generation, "v2|new-embed") + .await + .unwrap(); + + let stale_replace = store + .backfill_entity_vectors( + "default", + "Item", + "item-generation", + old_generation, + 1, + std::slice::from_ref(&old_row), + ) + .await; + assert!( + stale_replace.is_err(), + "an older declaration-set generation must not replace equal-sequence rows" + ); + let stale_watermark = store + .mark_vector_index_backfilled("default", "Item", old_generation, "v2|old-embed") + .await; + assert!( + stale_watermark.is_err(), + "an older declaration-set generation must not overwrite the newer watermark" + ); + assert!( + store + .vector_candidates("default", "Item", "old-embed", "m1", 10) + .await + .unwrap() + .is_empty() + ); + assert_eq!( + store + .vector_candidates("default", "Item", "new-embed", "m2", 10) + .await + .unwrap(), + vec![EntityVectorCandidate { + entity_id: "item-generation".to_string(), + vector: new_row.vector, + }] + ); + assert_eq!( + store + .vector_index_backfilled_types("default") + .await + .unwrap(), + vec![("Item".to_string(), "v2|new-embed".to_string())] + ); +} + +#[tokio::test] +async fn stale_declaration_cannot_reclaim_generation_after_newer_set_completes() { + let store = SimEventStore::no_faults(45); + let persistence_id = "default:Item:item-signature-race"; + let row_a = EntityVectorRow { + decl_name: "embed-a".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }; + let row_b = EntityVectorRow { + decl_name: "embed-b".to_string(), + model_tag: "m2".to_string(), + vector: vec![0.0, 1.0], + }; + + store.persist_spec_declaration("default", "Item", "rev-a"); + let first_a = store + .begin_vector_index_reconciliation("default", "Item", "v2|a", 1, "rev-a") + .await + .unwrap(); + store + .append_with_index_rows( + persistence_id, + 0, + &[test_envelope(0, "Created")], + &[], + std::slice::from_ref(&row_a), + true, + Some("rev-a"), + ) + .await + .unwrap(); + store + .mark_vector_index_backfilled("default", "Item", first_a, "v2|a") + .await + .unwrap(); + + store.persist_spec_declaration("default", "Item", "rev-b"); + let generation_b = store + .begin_vector_index_reconciliation("default", "Item", "v2|b", 2, "rev-b") + .await + .unwrap(); + assert!( + store + .vector_index_backfilled_types("default") + .await + .unwrap() + .is_empty(), + "beginning B must atomically withdraw A's completion watermark" + ); + assert_eq!( + store + .vector_reconciliation_entity_types("default") + .await + .unwrap(), + vec!["Item".to_string()], + "the in-progress type must remain discoverable without its watermark" + ); + + let stale_live = store + .append_with_index_rows( + persistence_id, + 1, + &[test_envelope(0, "StaleReplicaUpdated")], + &[], + std::slice::from_ref(&row_a), + true, + Some("rev-a"), + ) + .await; + assert!( + stale_live.is_err(), + "a stale replica must not advance the journal with rows from declaration A" + ); + assert_eq!(store.dump_journal(persistence_id).len(), 1); + + store + .append_with_index_rows( + persistence_id, + 1, + &[test_envelope(0, "CurrentReplicaUpdated")], + &[], + std::slice::from_ref(&row_b), + true, + Some("rev-b"), + ) + .await + .unwrap(); + + store + .backfill_entity_vectors( + "default", + "Item", + "item-signature-race", + generation_b, + 1, + std::slice::from_ref(&row_b), + ) + .await + .unwrap(); + store + .mark_vector_index_backfilled("default", "Item", generation_b, "v2|b") + .await + .unwrap(); + + // A stale replica still holding declaration set A must not obtain a later + // generation after the authoritative B revision has completed. + let stale_a = store + .begin_vector_index_reconciliation("default", "Item", "v2|a", 1, "rev-a") + .await; + assert!( + stale_a.is_err(), + "an older declaration revision must not reclaim authority by arriving last" + ); + assert_eq!( + store + .vector_index_backfilled_types("default") + .await + .unwrap(), + vec![("Item".to_string(), "v2|b".to_string())] + ); + assert_eq!( + store + .vector_candidates("default", "Item", "embed-b", "m2", 10) + .await + .unwrap(), + vec![EntityVectorCandidate { + entity_id: "item-signature-race".to_string(), + vector: row_b.vector, + }] + ); +} + +#[tokio::test] +async fn caller_local_revision_cannot_override_durable_declaration_authority() { + let store = SimEventStore::no_faults(48); + store.persist_spec_declaration("default", "Item", "rev-a"); + let generation_a = store + .begin_vector_index_reconciliation("default", "Item", "v2|a", 1, "rev-a") + .await + .unwrap(); + store + .mark_vector_index_backfilled("default", "Item", generation_a, "v2|a") + .await + .unwrap(); + + store.persist_spec_declaration("default", "Item", "rev-b"); + let generation_b = store + .begin_vector_index_reconciliation("default", "Item", "v2|b", 1, "rev-b") + .await + .unwrap(); + store + .mark_vector_index_backfilled("default", "Item", generation_b, "v2|b") + .await + .unwrap(); + + let stale = store + .begin_vector_index_reconciliation("default", "Item", "v2|a", u64::MAX, "rev-a") + .await; + assert!( + stale.is_err(), + "even a maximal caller-local revision must not replace persisted B authority" + ); + assert_eq!( + store + .vector_index_backfilled_types("default") + .await + .unwrap(), + vec![("Item".to_string(), "v2|b".to_string())] + ); +} + +#[tokio::test] +async fn fresh_reconciliation_ignores_maximal_caller_revision() { + let store = SimEventStore::no_faults(52); + let first_generation = store + .begin_vector_index_reconciliation("default", "Item", "v2|a", u64::MAX, "rev-a") + .await + .expect("bootstrap fresh declaration authority"); + assert_eq!(first_generation, 1); + + let next_revision = store.persist_spec_declaration("default", "Item", "rev-b"); + assert_eq!( + next_revision, 2, + "durable authority must start at revision one" + ); + assert!( + store + .begin_vector_index_reconciliation("default", "Item", "v2|a", 1, "rev-a") + .await + .is_err(), + "the next persisted declaration must fence the fresh generation" + ); + assert_eq!( + store + .begin_vector_index_reconciliation("default", "Item", "v2|b", 1, "rev-b") + .await + .expect("begin next declaration"), + 2 + ); +} + +#[tokio::test] +async fn rejected_single_append_does_not_publish_bootstrapped_authority() { + let store = SimEventStore::no_faults(49); + let claimed_key = temper_runtime::persistence::EntityKeyRow { + key_name: "external-id".to_string(), + key_hash: "shared-key".to_string(), + }; + store + .append_with_index_rows( + "default:Item:owner", + 0, + &[test_envelope(0, "Created")], + std::slice::from_ref(&claimed_key), + &[], + false, + None, + ) + .await + .unwrap(); + + let rejected = store + .append_with_index_rows( + "default:Item:contender", + 0, + &[test_envelope(0, "Created")], + std::slice::from_ref(&claimed_key), + &[], + true, + Some("rev-a"), + ) + .await; + assert!(rejected.is_err(), "duplicate key must reject the append"); + assert!(store.dump_journal("default:Item:contender").is_empty()); + assert!( + store + .vector_reconciliation_entity_types("default") + .await + .unwrap() + .is_empty(), + "a rejected append must not leak a generation-zero work row" + ); + + let generation = store + .begin_vector_index_reconciliation("default", "Item", "v2|b", 1, "rev-b") + .await + .expect("the rejected rev-a bootstrap must not become durable authority"); + assert_eq!(generation, 1); +} + +#[tokio::test] +async fn rejected_batch_does_not_publish_bootstrapped_authority() { + let store = SimEventStore::no_faults(50); + let rejected = store + .append_batch(&[ + PersistenceAppend { + persistence_id: "default:Item:first".to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "Created")], + vector_rows: Vec::new(), + reconcile_vectors: true, + spec_declaration_fingerprint: Some("rev-a".to_string()), + }, + PersistenceAppend { + persistence_id: "malformed".to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "Created")], + vector_rows: Vec::new(), + reconcile_vectors: true, + spec_declaration_fingerprint: Some("rev-a".to_string()), + }, + ]) + .await; + assert!( + rejected.is_err(), + "malformed second stream must abort the batch" + ); + assert!(store.dump_journal("default:Item:first").is_empty()); + assert!(store.dump_journal("malformed").is_empty()); + assert!( + store + .vector_reconciliation_entity_types("default") + .await + .unwrap() + .is_empty(), + "an aborted batch must not leak authority-derived work" + ); + + let generation = store + .begin_vector_index_reconciliation("default", "Item", "v2|b", 1, "rev-b") + .await + .expect("the aborted rev-a batch must not become durable authority"); + assert_eq!(generation, 1); +} + +#[tokio::test] +async fn deleted_declaration_reconciliation_resumes_after_store_restart() { + let store = SimEventStore::no_faults(46); + store.persist_spec_declaration("default", "Item", "rev-a"); + let present_generation = store + .begin_vector_index_reconciliation("default", "Item", "v2|embed", 1, "rev-a") + .await + .unwrap(); + store + .mark_vector_index_backfilled("default", "Item", present_generation, "v2|embed") + .await + .unwrap(); + + store.persist_spec_declaration("default", "Item", "absent:v1"); + let absent_generation = store + .begin_vector_index_reconciliation("default", "Item", "v2|", 2, "absent:v1") + .await + .unwrap(); + assert!(absent_generation > present_generation); + assert!( + store + .vector_index_backfilled_types("default") + .await + .unwrap() + .is_empty(), + "starting the deletion purge must withdraw the old watermark" + ); + + // A reopened handle retains durable authority while a rebuilt process-local + // registry restarts its diagnostic revision at one. + let restarted = store.clone(); + drop(store); + let resumed_generation = restarted + .begin_vector_index_reconciliation("default", "Item", "v2|", 1, "absent:v1") + .await + .unwrap(); + assert_eq!(resumed_generation, absent_generation); + restarted + .mark_vector_index_backfilled("default", "Item", resumed_generation, "v2|") + .await + .unwrap(); + + restarted.persist_spec_declaration("default", "Item", "rev-a"); + let readded_generation = restarted + .begin_vector_index_reconciliation("default", "Item", "v2|embed", 3, "rev-a") + .await + .unwrap(); + assert!( + readded_generation > resumed_generation, + "an identical declaration re-add must remain a newer authority revision" + ); +} + +#[tokio::test] +async fn composite_batch_vector_fence_rejects_delayed_repair() { + let store = SimEventStore::no_faults(44); + let persistence_id = "default:Item:item-composite"; + store.persist_spec_declaration("default", "Item", "rev-1"); + let generation = store + .begin_vector_index_reconciliation("default", "Item", "v2|embed", 1, "rev-1") + .await + .unwrap(); + let stale_row = EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }; + let live_row = EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![0.0, 1.0], + }; + + store + .append_with_index_rows( + persistence_id, + 0, + &[test_envelope(0, "Created")], + &[], + std::slice::from_ref(&stale_row), + true, + Some("rev-1"), + ) + .await + .unwrap(); + store + .append_batch(&[PersistenceAppend { + persistence_id: persistence_id.to_string(), + expected_sequence: 1, + events: vec![test_envelope(0, "CompositeUpdated")], + vector_rows: vec![live_row.clone()], + reconcile_vectors: true, + spec_declaration_fingerprint: Some("rev-1".to_string()), + }]) + .await + .unwrap(); + store + .backfill_entity_vectors( + "default", + "Item", + "item-composite", + generation, + 1, + std::slice::from_ref(&stale_row), + ) + .await + .unwrap(); + assert_eq!( + store + .vector_candidates("default", "Item", "embed", "m1", 10) + .await + .unwrap()[0] + .vector, + live_row.vector.clone() + ); + + store + .append_batch(&[PersistenceAppend { + persistence_id: persistence_id.to_string(), + expected_sequence: 2, + events: vec![test_envelope(0, "CompositeDeleted")], + vector_rows: Vec::new(), + reconcile_vectors: true, + spec_declaration_fingerprint: Some("rev-1".to_string()), + }]) + .await + .unwrap(); + store + .backfill_entity_vectors( + "default", + "Item", + "item-composite", + generation, + 2, + std::slice::from_ref(&live_row), + ) + .await + .unwrap(); + assert!( + store + .vector_candidates("default", "Item", "embed", "m1", 10) + .await + .unwrap() + .is_empty(), + "the composite delete's sequence-3 fence must reject sequence-2 resurrection" + ); + assert_eq!(store.dump_journal(persistence_id).len(), 3); +} + +#[tokio::test] +async fn append_batch_commits_multiple_journals_atomically() { + let store = SimEventStore::no_faults(42); + let appends = vec![ + PersistenceAppend { + persistence_id: "default:Order:ord-a".to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "Created")], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, + }, + PersistenceAppend { + persistence_id: "default:Order:ord-b".to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "Created"), test_envelope(0, "Submitted")], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, + }, + ]; + + let results = store.append_batch(&appends).await.unwrap(); + + assert_eq!( + results, + vec![ + PersistenceAppendResult { + persistence_id: "default:Order:ord-a".to_string(), + sequence_nr: 1, + }, + PersistenceAppendResult { + persistence_id: "default:Order:ord-b".to_string(), + sequence_nr: 2, + }, + ] + ); + assert_eq!(store.dump_journal("default:Order:ord-a").len(), 1); + assert_eq!(store.dump_journal("default:Order:ord-b").len(), 2); +} + +#[tokio::test] +async fn append_batch_conflict_leaves_all_journals_untouched() { + let store = SimEventStore::no_faults(42); + store + .append( + "default:Order:ord-existing", + 0, + &[test_envelope(0, "Created")], + ) + .await + .unwrap(); + + let err = store + .append_batch(&[ + PersistenceAppend { + persistence_id: "default:Order:ord-new".to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "Created")], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, + }, + PersistenceAppend { + persistence_id: "default:Order:ord-existing".to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "Submitted")], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, + }, + ]) + .await + .expect_err("second journal conflict should abort entire batch"); + + assert!( + matches!(err, PersistenceError::ConcurrencyViolation { .. }), + "unexpected error: {err}" + ); + assert!( + store.dump_journal("default:Order:ord-new").is_empty(), + "first append must not be persisted when a later stream conflicts" + ); + assert_eq!( + store.dump_journal("default:Order:ord-existing").len(), + 1, + "conflicting stream must keep its original journal only" + ); +} + +#[tokio::test] +async fn append_batch_preflight_reports_exact_sequence_without_consuming_fault() { + let store = SimEventStore::no_faults(42); + let existing = "default:Order:ord-batch-existing"; + let new = "default:Order:ord-batch-new"; + store + .append( + existing, + 0, + &[test_envelope(0, "Created"), test_envelope(0, "Submitted")], + ) + .await + .unwrap(); + store.inject_concurrency_violations(existing, 1); + + let error = store + .append_batch(&[ + PersistenceAppend { + persistence_id: new.to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "Created")], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, + }, + PersistenceAppend { + persistence_id: existing.to_string(), + expected_sequence: 99, + events: vec![test_envelope(0, "Duplicate")], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, + }, + ]) + .await + .expect_err("stale batch must fail before consuming its injected fault"); + assert!(matches!( + error, + PersistenceError::ConcurrencyViolation { + expected: 99, + actual: 2 + } + )); + assert_eq!(store.pending_concurrency_violations(existing), 1); + assert!(store.dump_journal(new).is_empty()); + assert_eq!(store.dump_journal(existing).len(), 2); + + let injected = store + .append_batch(&[PersistenceAppend { + persistence_id: existing.to_string(), + expected_sequence: 2, + events: vec![test_envelope(0, "Injected")], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, + }]) + .await + .expect_err("the preserved injected fault must reject the next valid batch"); + assert!(matches!( + injected, + PersistenceError::ConcurrencyViolation { + expected: 2, + actual: 2 + } + )); + assert_eq!(store.pending_concurrency_violations(existing), 0); + assert_eq!(store.dump_journal(existing).len(), 2); +} + +#[tokio::test] +async fn probabilistic_append_batch_reports_unchanged_durable_sequence() { + let store = SimEventStore::new( + 42, + SimFaultConfig { + write_failure_prob: 0.0, + concurrency_violation_prob: 1.0, + read_truncation_prob: 0.0, + snapshot_failure_prob: 0.0, + }, + ); + let persistence_id = "default:Order:probabilistic-batch-conflict"; + + let error = store + .append_batch(&[PersistenceAppend { + persistence_id: persistence_id.to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "Created")], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, + }]) + .await + .expect_err("probabilistic concurrency fault must reject the batch"); + assert!(matches!( + error, + PersistenceError::ConcurrencyViolation { + expected: 0, + actual: 0 + } + )); + assert!(store.dump_journal(persistence_id).is_empty()); +} + +#[tokio::test] +async fn concurrency_violation_on_wrong_sequence() { + let store = SimEventStore::no_faults(42); + let pid = "default:Order:ord-3"; + + store + .append(pid, 0, &[test_envelope(0, "Created")]) + .await + .unwrap(); + + let err = store + .append(pid, 0, &[test_envelope(0, "Duplicate")]) + .await + .unwrap_err(); + + assert!(matches!( + err, + PersistenceError::ConcurrencyViolation { + expected: 0, + actual: 1 + } + )); +} + +#[tokio::test] +async fn injected_concurrency_violation_reports_durable_sequence() { + let store = SimEventStore::new( + 42, + SimFaultConfig { + write_failure_prob: 0.0, + concurrency_violation_prob: 1.0, + read_truncation_prob: 0.0, + snapshot_failure_prob: 0.0, + }, + ); + let pid = "default:Order:injected-conflict"; + + let error = store + .append(pid, 0, &[test_envelope(0, "Created")]) + .await + .expect_err("injected concurrency violation should reject the append"); + + match error { + PersistenceError::ConcurrencyViolation { expected, actual } => assert_eq!( + (expected, actual), + (0, 0), + "the reported authoritative sequence must match the unchanged journal" + ), + other => panic!("unexpected injected error: {other}"), + } + assert!(store.dump_journal(pid).is_empty()); +} + +#[tokio::test] +async fn snapshot_save_and_load() { + let store = SimEventStore::no_faults(42); + let pid = "default:Order:ord-4"; + + store.save_snapshot(pid, 5, b"state-data").await.unwrap(); + + let snap = store.load_snapshot(pid).await.unwrap(); + assert_eq!(snap, Some((5, b"state-data".to_vec()))); +} + +#[tokio::test] +async fn snapshot_save_records_history_and_rotates_segments() { + let store = SimEventStore::no_faults(42); + let pid = "default:Order:segmented"; + + store + .append( + pid, + 0, + &[test_envelope(0, "Created"), test_envelope(0, "Updated")], + ) + .await + .unwrap(); + store.save_snapshot(pid, 2, b"snapshot-2").await.unwrap(); + store + .append(pid, 2, &[test_envelope(0, "AfterSnapshot")]) + .await + .unwrap(); + + assert_eq!(store.snapshot_history_len(pid), 1); + let segments = store.dump_segments(pid); + assert_eq!(segments.len(), 2); + assert_eq!(segments[0].segment_index, 0); + assert_eq!(segments[0].snapshot_sequence, Some(2)); + assert!(segments[0].sealed); + assert_eq!(segments[1].segment_index, 1); + assert_eq!(segments[1].start_sequence_nr, 3); + assert_eq!(segments[1].end_sequence_nr, Some(3)); + assert!(!segments[1].sealed); +} + +#[tokio::test] +async fn load_snapshot_returns_none_when_empty() { + let store = SimEventStore::no_faults(42); + let snap = store + .load_snapshot("default:Order:nonexistent") + .await + .unwrap(); + assert_eq!(snap, None); +} + +#[tokio::test] +async fn list_entity_ids_filters_by_tenant() { + let store = SimEventStore::no_faults(42); + + store + .append("alpha:Order:ord-1", 0, &[test_envelope(0, "Created")]) + .await + .unwrap(); + store + .append("alpha:Task:task-1", 0, &[test_envelope(0, "Created")]) + .await + .unwrap(); + store + .append("beta:Order:ord-9", 0, &[test_envelope(0, "Created")]) + .await + .unwrap(); + + let mut alpha = store.list_entity_ids("alpha").await.unwrap(); + alpha.sort(); + assert_eq!( + alpha, + vec![ + ("Order".to_string(), "ord-1".to_string()), + ("Task".to_string(), "task-1".to_string()), + ] + ); + + let beta = store.list_entity_ids("beta").await.unwrap(); + assert_eq!(beta, vec![("Order".to_string(), "ord-9".to_string())]); +} + +#[tokio::test] +async fn read_events_from_sequence() { + let store = SimEventStore::no_faults(42); + let pid = "default:Order:ord-5"; + + store + .append(pid, 0, &[test_envelope(0, "A"), test_envelope(0, "B")]) + .await + .unwrap(); + store + .append(pid, 2, &[test_envelope(0, "C")]) + .await + .unwrap(); + + // Read from sequence 1 — should skip event at seq 1 + let events = store.read_events(pid, 1).await.unwrap(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].sequence_nr, 2); + assert_eq!(events[1].sequence_nr, 3); +} + +#[tokio::test] +async fn deterministic_across_seeds() { + // Same seed → same behavior (with no faults, behavior is trivially the same) + for seed in [42, 123, 999] { + let store = SimEventStore::no_faults(seed); + let pid = "default:Order:det-1"; + + let seq = store + .append(pid, 0, &[test_envelope(0, "Created")]) + .await + .unwrap(); + assert_eq!(seq, 1); + + let events = store.read_events(pid, 0).await.unwrap(); + assert_eq!(events.len(), 1); + } +} + +#[tokio::test] +async fn fault_injection_produces_errors() { + let faults = SimFaultConfig { + write_failure_prob: 1.0, // always fail + concurrency_violation_prob: 0.0, + read_truncation_prob: 0.0, + snapshot_failure_prob: 0.0, + }; + let store = SimEventStore::new(42, faults); + let pid = "default:Order:fault-1"; + + let err = store.append(pid, 0, &[test_envelope(0, "Created")]).await; + assert!(err.is_err()); +} diff --git a/crates/temper-store-turso/src/router.rs b/crates/temper-store-turso/src/router.rs index 3c7e1fe01..6c9ec2038 100644 --- a/crates/temper-store-turso/src/router.rs +++ b/crates/temper-store-turso/src/router.rs @@ -740,6 +740,18 @@ impl EventStore for TenantStoreRouter { store.list_entity_ids_by_type(tenant, entity_type).await } + #[instrument(skip_all, fields(tenant, entity_type, otel.name = "router.list_vector_repair_entity_ids"))] + async fn list_vector_repair_entity_ids( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + let store = self.store_for_tenant(tenant).await?; + store + .list_vector_repair_entity_ids(tenant, entity_type) + .await + } + // ADR-0155: forward the vector-index surface to the per-tenant store so kNN works // on the routed Turso deployment. (Keys deliberately fall through to the no-op // defaults — Turso does not maintain entity_key_index live; see event_store.rs.) @@ -752,6 +764,7 @@ impl EventStore for TenantStoreRouter { key_rows: &[temper_runtime::persistence::EntityKeyRow], vector_rows: &[temper_runtime::persistence::EntityVectorRow], reconcile_vectors: bool, + spec_declaration_fingerprint: Option<&str>, ) -> Result { let (tenant, _, _) = parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; @@ -764,6 +777,7 @@ impl EventStore for TenantStoreRouter { key_rows, vector_rows, reconcile_vectors, + spec_declaration_fingerprint, ) .await } @@ -774,11 +788,41 @@ impl EventStore for TenantStoreRouter { tenant: &str, entity_type: &str, entity_id: &str, + reconciliation_generation: u64, + observed_sequence: u64, vector_rows: &[temper_runtime::persistence::EntityVectorRow], ) -> Result<(), PersistenceError> { let store = self.store_for_tenant(tenant).await?; store - .backfill_entity_vectors(tenant, entity_type, entity_id, vector_rows) + .backfill_entity_vectors( + tenant, + entity_type, + entity_id, + reconciliation_generation, + observed_sequence, + vector_rows, + ) + .await + } + + #[instrument(skip_all, fields(tenant, entity_type, otel.name = "router.begin_vector_index_reconciliation"))] + async fn begin_vector_index_reconciliation( + &self, + tenant: &str, + entity_type: &str, + vector_set: &str, + declaration_revision: u64, + declaration_fingerprint: &str, + ) -> Result { + let store = self.store_for_tenant(tenant).await?; + store + .begin_vector_index_reconciliation( + tenant, + entity_type, + vector_set, + declaration_revision, + declaration_fingerprint, + ) .await } @@ -802,11 +846,17 @@ impl EventStore for TenantStoreRouter { &self, tenant: &str, entity_type: &str, + reconciliation_generation: u64, vector_set: &str, ) -> Result<(), PersistenceError> { let store = self.store_for_tenant(tenant).await?; store - .mark_vector_index_backfilled(tenant, entity_type, vector_set) + .mark_vector_index_backfilled( + tenant, + entity_type, + reconciliation_generation, + vector_set, + ) .await } @@ -819,6 +869,15 @@ impl EventStore for TenantStoreRouter { store.vector_index_backfilled_types(tenant).await } + #[instrument(skip_all, fields(tenant, otel.name = "router.vector_reconciliation_entity_types"))] + async fn vector_reconciliation_entity_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + let store = self.store_for_tenant(tenant).await?; + store.vector_reconciliation_entity_types(tenant).await + } + #[instrument(skip_all, fields(tenant, entity_type, otel.name = "router.vectored_entity_ids_for_type"))] async fn vectored_entity_ids_for_type( &self, diff --git a/crates/temper-store-turso/src/schema.rs b/crates/temper-store-turso/src/schema.rs index 1012b2e03..ea5c0fa12 100644 --- a/crates/temper-store-turso/src/schema.rs +++ b/crates/temper-store-turso/src/schema.rs @@ -1,18 +1,20 @@ //! SQLite-compatible schema for the Turso/libSQL event store. - -mod query_plane; +pub(crate) mod declaration_authority; +pub(crate) mod query_plane; pub use crate::schema_event_history::{ ALTER_EVENTS_ADD_SEGMENT_INDEX, CREATE_EVENT_SEGMENTS_OPEN_INDEX, CREATE_EVENT_SEGMENTS_TABLE, CREATE_SNAPSHOT_HISTORY_ENTITY_INDEX, CREATE_SNAPSHOT_HISTORY_TABLE, }; pub use query_plane::{ - CREATE_ENTITY_CATALOG_STATUS_INDEX, CREATE_ENTITY_CATALOG_TABLE, - CREATE_ENTITY_CATALOG_TYPE_INDEX, CREATE_ENTITY_FIELD_INDEX_LOOKUP, - CREATE_ENTITY_FIELD_INDEX_STATUS, CREATE_ENTITY_FIELD_INDEX_TABLE, - CREATE_ENTITY_KEY_INDEX_ENTITY, CREATE_ENTITY_KEY_INDEX_TABLE, + ALTER_ENTITY_VECTOR_INDEX_VERSION_ADD_GENERATION, CREATE_ENTITY_CATALOG_STATUS_INDEX, + CREATE_ENTITY_CATALOG_TABLE, CREATE_ENTITY_CATALOG_TYPE_INDEX, + CREATE_ENTITY_FIELD_INDEX_LOOKUP, CREATE_ENTITY_FIELD_INDEX_STATUS, + CREATE_ENTITY_FIELD_INDEX_TABLE, CREATE_ENTITY_KEY_INDEX_ENTITY, CREATE_ENTITY_KEY_INDEX_TABLE, CREATE_ENTITY_VECTOR_INDEX_ENTITY, CREATE_ENTITY_VECTOR_INDEX_PARTITION, - CREATE_ENTITY_VECTOR_INDEX_TABLE, CREATE_VECTOR_INDEX_BACKFILL_WATERMARK, + CREATE_ENTITY_VECTOR_INDEX_TABLE, CREATE_ENTITY_VECTOR_INDEX_VERSION_TABLE, + CREATE_VECTOR_INDEX_BACKFILL_WATERMARK, CREATE_VECTOR_RECONCILIATION_GENERATION_TABLE, + SEED_ENTITY_VECTOR_INDEX_VERSION_TABLE, }; pub const CREATE_EVENTS_TABLE: &str = "\ @@ -61,6 +63,21 @@ CREATE TABLE IF NOT EXISTS specs ( UNIQUE(tenant, entity_type) );"; +/// Replacement spec bytes awaiting verification. +/// +/// Keeping staging separate preserves the last committed catalog across crashes. +pub const CREATE_STAGED_SPECS_TABLE: &str = "\ +CREATE TABLE IF NOT EXISTS staged_specs ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + ioa_source TEXT NOT NULL, + csdl_xml TEXT, + content_hash TEXT NOT NULL, + version INTEGER NOT NULL DEFAULT 1, + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(tenant, entity_type) +);"; + pub const CREATE_TRAJECTORIES_TABLE: &str = "\ CREATE TABLE IF NOT EXISTS trajectories ( id INTEGER PRIMARY KEY AUTOINCREMENT, diff --git a/crates/temper-store-turso/src/schema/declaration_authority.rs b/crates/temper-store-turso/src/schema/declaration_authority.rs new file mode 100644 index 000000000..5cbed9f1c --- /dev/null +++ b/crates/temper-store-turso/src/schema/declaration_authority.rs @@ -0,0 +1,154 @@ +//! Durable spec-declaration ordering used by vector reconciliation (ADR-0181). + +/// Per-type monotonic declaration source/tombstone, independent of vector work. +pub(crate) const CREATE_SPEC_DECLARATION_AUTHORITY_TABLE: &str = "\ +CREATE TABLE IF NOT EXISTS spec_declaration_authority ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + revision INTEGER NOT NULL, + ioa_source TEXT NOT NULL DEFAULT '', + declaration_fingerprint TEXT NOT NULL DEFAULT '', + present INTEGER NOT NULL, + PRIMARY KEY (tenant, entity_type) +);"; + +/// Upgrade authority tables created by an earlier ADR-0181 build. +pub(crate) const ALTER_SPEC_DECLARATION_AUTHORITY_ADD_FINGERPRINT: &str = "\ +ALTER TABLE spec_declaration_authority +ADD COLUMN declaration_fingerprint TEXT NOT NULL DEFAULT '';"; + +/// Bootstrap authority for specs that exist before the ADR-0181 triggers. +const SEED_PRESENT_SPEC_DECLARATION_AUTHORITY: &str = "\ +INSERT INTO spec_declaration_authority + (tenant, entity_type, revision, ioa_source, declaration_fingerprint, present) +SELECT tenant, entity_type, MAX(version, 1), ioa_source, COALESCE(content_hash, ''), 1 +FROM specs +WHERE committed = 1 +ON CONFLICT(tenant, entity_type) DO NOTHING;"; + +/// Bootstrap deletion tombstones for retained legacy vector state. +const SEED_ABSENT_SPEC_DECLARATION_AUTHORITY: &str = "\ +INSERT INTO spec_declaration_authority + (tenant, entity_type, revision, ioa_source, declaration_fingerprint, present) +SELECT known.tenant, known.entity_type, 1, '', 'absent:v1', 0 +FROM ( + SELECT tenant, entity_type FROM entity_vector_index + UNION + SELECT tenant, entity_type FROM entity_vector_index_version + UNION + SELECT tenant, entity_type FROM entity_vector_reconciliation_generation + UNION + SELECT tenant, entity_type FROM vector_index_backfill_watermark +) AS known +WHERE NOT EXISTS ( + SELECT 1 + FROM specs + WHERE specs.tenant = known.tenant + AND specs.entity_type = known.entity_type + AND specs.committed = 1 +) +ON CONFLICT(tenant, entity_type) DO NOTHING;"; + +/// Advance declaration authority and fence vector work on committed spec insert. +const DROP_SPEC_DECLARATION_INSERT_TRIGGER: &str = + "DROP TRIGGER IF EXISTS specs_declaration_authority_insert;"; +const CREATE_SPEC_DECLARATION_INSERT_TRIGGER: &str = "\ +CREATE TRIGGER specs_declaration_authority_insert +AFTER INSERT ON specs +WHEN NEW.committed = 1 +BEGIN + INSERT INTO spec_declaration_authority + (tenant, entity_type, revision, ioa_source, declaration_fingerprint, present) + VALUES (NEW.tenant, NEW.entity_type, 1, NEW.ioa_source, COALESCE(NEW.content_hash, ''), 1) + ON CONFLICT(tenant, entity_type) DO UPDATE SET + revision = spec_declaration_authority.revision + 1, + ioa_source = excluded.ioa_source, + declaration_fingerprint = excluded.declaration_fingerprint, + present = excluded.present; + UPDATE entity_vector_reconciliation_generation + SET generation = generation + 1, + declaration_revision = ( + SELECT revision FROM spec_declaration_authority + WHERE tenant = NEW.tenant AND entity_type = NEW.entity_type + ), + declaration_fingerprint = '', + vector_set = '' + WHERE tenant = NEW.tenant AND entity_type = NEW.entity_type; + DELETE FROM vector_index_backfill_watermark + WHERE tenant = NEW.tenant AND entity_type = NEW.entity_type; +END;"; + +/// Advance authority when a staged spec commits or committed content changes. +const DROP_SPEC_DECLARATION_UPDATE_TRIGGER: &str = + "DROP TRIGGER IF EXISTS specs_declaration_authority_update;"; +const CREATE_SPEC_DECLARATION_UPDATE_TRIGGER: &str = "\ +CREATE TRIGGER specs_declaration_authority_update +AFTER UPDATE OF ioa_source, content_hash, committed ON specs +WHEN NEW.committed = 1 + AND ( + OLD.committed IS NOT NEW.committed + OR OLD.ioa_source IS NOT NEW.ioa_source + OR OLD.content_hash IS NOT NEW.content_hash + ) +BEGIN + INSERT INTO spec_declaration_authority + (tenant, entity_type, revision, ioa_source, declaration_fingerprint, present) + VALUES (NEW.tenant, NEW.entity_type, 1, NEW.ioa_source, COALESCE(NEW.content_hash, ''), 1) + ON CONFLICT(tenant, entity_type) DO UPDATE SET + revision = spec_declaration_authority.revision + 1, + ioa_source = excluded.ioa_source, + declaration_fingerprint = excluded.declaration_fingerprint, + present = excluded.present; + UPDATE entity_vector_reconciliation_generation + SET generation = generation + 1, + declaration_revision = ( + SELECT revision FROM spec_declaration_authority + WHERE tenant = NEW.tenant AND entity_type = NEW.entity_type + ), + declaration_fingerprint = '', + vector_set = '' + WHERE tenant = NEW.tenant AND entity_type = NEW.entity_type; + DELETE FROM vector_index_backfill_watermark + WHERE tenant = NEW.tenant AND entity_type = NEW.entity_type; +END;"; + +/// Tombstone only deletion of a committed spec; staged cleanup is invisible. +const DROP_SPEC_DECLARATION_DELETE_TRIGGER: &str = + "DROP TRIGGER IF EXISTS specs_declaration_authority_delete;"; +const CREATE_SPEC_DECLARATION_DELETE_TRIGGER: &str = "\ +CREATE TRIGGER specs_declaration_authority_delete +AFTER DELETE ON specs +WHEN OLD.committed = 1 +BEGIN + INSERT INTO spec_declaration_authority + (tenant, entity_type, revision, ioa_source, declaration_fingerprint, present) + VALUES (OLD.tenant, OLD.entity_type, 1, '', 'absent:v1', 0) + ON CONFLICT(tenant, entity_type) DO UPDATE SET + revision = spec_declaration_authority.revision + 1, + ioa_source = excluded.ioa_source, + declaration_fingerprint = excluded.declaration_fingerprint, + present = excluded.present; + UPDATE entity_vector_reconciliation_generation + SET generation = generation + 1, + declaration_revision = ( + SELECT revision FROM spec_declaration_authority + WHERE tenant = OLD.tenant AND entity_type = OLD.entity_type + ), + declaration_fingerprint = '', + vector_set = '' + WHERE tenant = OLD.tenant AND entity_type = OLD.entity_type; + DELETE FROM vector_index_backfill_watermark + WHERE tenant = OLD.tenant AND entity_type = OLD.entity_type; +END;"; + +/// Ordered schema statements for durable declaration authority. +pub(crate) const DECLARATION_AUTHORITY_STATEMENTS: &[&str] = &[ + SEED_PRESENT_SPEC_DECLARATION_AUTHORITY, + SEED_ABSENT_SPEC_DECLARATION_AUTHORITY, + DROP_SPEC_DECLARATION_INSERT_TRIGGER, + CREATE_SPEC_DECLARATION_INSERT_TRIGGER, + DROP_SPEC_DECLARATION_UPDATE_TRIGGER, + CREATE_SPEC_DECLARATION_UPDATE_TRIGGER, + DROP_SPEC_DECLARATION_DELETE_TRIGGER, + CREATE_SPEC_DECLARATION_DELETE_TRIGGER, +]; diff --git a/crates/temper-store-turso/src/schema/query_plane.rs b/crates/temper-store-turso/src/schema/query_plane.rs index 827e3b8df..f84591675 100644 --- a/crates/temper-store-turso/src/schema/query_plane.rs +++ b/crates/temper-store-turso/src/schema/query_plane.rs @@ -77,10 +77,8 @@ CREATE INDEX IF NOT EXISTS idx_eki_entity /// ADR-0155: declared vector access path — the exact-scan kNN index. One row per /// (declared vector path, model tag, entity). `vector` is packed little-endian -/// f32; `model_tag` partitions the space. Unlike keys, Turso maintains this -/// **write-behind** (the event append is followed by the index write, not -/// co-committed) — safe because a vector row carries no uniqueness constraint; the -/// backfill watermark gates when the index is authoritatively complete. +/// f32; `model_tag` partitions the space. Turso co-commits these rows and their +/// retained per-entity sequence fence with the journal append (ADR-0181). pub const CREATE_ENTITY_VECTOR_INDEX_TABLE: &str = "\ CREATE TABLE IF NOT EXISTS entity_vector_index ( tenant TEXT NOT NULL, @@ -104,6 +102,66 @@ pub const CREATE_ENTITY_VECTOR_INDEX_ENTITY: &str = "\ CREATE INDEX IF NOT EXISTS idx_evi_entity ON entity_vector_index(tenant, entity_type, entity_id);"; +/// ADR-0181 retained per-entity vector reconciliation fence. This row remains even +/// when reconciliation produces no vector rows, preventing stale resurrection. +pub const CREATE_ENTITY_VECTOR_INDEX_VERSION_TABLE: &str = "\ +CREATE TABLE IF NOT EXISTS entity_vector_index_version ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id TEXT NOT NULL, + reconciliation_generation INTEGER NOT NULL DEFAULT 0, + sequence_nr INTEGER NOT NULL, + PRIMARY KEY (tenant, entity_type, entity_id) +);"; + +/// Idempotent-at-bootstrap upgrade for databases created before ADR-0181 gained +/// declaration-set generations. Duplicate-column errors are ignored by the caller. +pub const ALTER_ENTITY_VECTOR_INDEX_VERSION_ADD_GENERATION: &str = "\ +ALTER TABLE entity_vector_index_version +ADD COLUMN reconciliation_generation INTEGER NOT NULL DEFAULT 0"; + +/// Durable ordering token for overlapping declaration-set reconciliation. +pub const CREATE_VECTOR_RECONCILIATION_GENERATION_TABLE: &str = "\ +CREATE TABLE IF NOT EXISTS entity_vector_reconciliation_generation ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + generation INTEGER NOT NULL, + declaration_revision INTEGER NOT NULL DEFAULT 0, + declaration_fingerprint TEXT NOT NULL DEFAULT '', + vector_set TEXT NOT NULL, + PRIMARY KEY (tenant, entity_type) +);"; + +/// Idempotent-at-bootstrap declaration revision upgrade for ADR-0181 databases. +pub const ALTER_VECTOR_RECONCILIATION_ADD_DECLARATION_REVISION: &str = "\ +ALTER TABLE entity_vector_reconciliation_generation +ADD COLUMN declaration_revision INTEGER NOT NULL DEFAULT 0"; + +/// Idempotent-at-bootstrap declaration fingerprint upgrade for ADR-0181 databases. +pub const ALTER_VECTOR_RECONCILIATION_ADD_DECLARATION_FINGERPRINT: &str = "\ +ALTER TABLE entity_vector_reconciliation_generation +ADD COLUMN declaration_fingerprint TEXT NOT NULL DEFAULT ''"; + +/// Seed the retained fence when upgrading a database that already has vector rows. +pub const SEED_ENTITY_VECTOR_INDEX_VERSION_TABLE: &str = "\ +INSERT INTO entity_vector_index_version + (tenant, entity_type, entity_id, reconciliation_generation, sequence_nr) +SELECT tenant, entity_type, entity_id, 0, MAX(sequence_nr) +FROM entity_vector_index +GROUP BY tenant, entity_type, entity_id +ON CONFLICT(tenant, entity_type, entity_id) +DO UPDATE SET + reconciliation_generation = MAX( + entity_vector_index_version.reconciliation_generation, + excluded.reconciliation_generation + ), + sequence_nr = CASE + WHEN entity_vector_index_version.reconciliation_generation + = excluded.reconciliation_generation + THEN MAX(entity_vector_index_version.sequence_nr, excluded.sequence_nr) + ELSE entity_vector_index_version.sequence_nr + END;"; + /// Per-(tenant, entity_type) vector-index backfill watermark (ADR-0155): records /// the covered vector-path set so a keyed read knows when the index is complete and /// re-indexes on a set change. Mirrors `key_index_backfill_watermark`. diff --git a/crates/temper-store-turso/src/store/event_store.rs b/crates/temper-store-turso/src/store/event_store.rs index 84d23fb04..6508b5d93 100644 --- a/crates/temper-store-turso/src/store/event_store.rs +++ b/crates/temper-store-turso/src/store/event_store.rs @@ -8,7 +8,7 @@ use temper_runtime::persistence::{ unpack_f32_le, }; use temper_runtime::tenant::parse_persistence_id_parts; -use tracing::{error, instrument, warn}; +use tracing::{instrument, warn}; use super::TursoEventStore; use super::append_config::{append_attempt_timeout, append_max_attempts}; @@ -18,6 +18,7 @@ use crate::metrics::record_turso_write_retry; use crate::retry::{is_transient_write_error, retry_delay_ms}; const APPEND_BATCH_INSERT_CHUNK_ROWS: usize = 400; +const ABSENT_DECLARATION_FINGERPRINT: &str = "absent:v1"; struct PreparedEventInsert { tenant: String, @@ -30,6 +31,177 @@ struct PreparedEventInsert { expected_sequence: u64, } +async fn current_vector_generation( + tx: &libsql::Transaction, + tenant: &str, + entity_type: &str, +) -> Result { + tx.execute( + "INSERT INTO entity_vector_reconciliation_generation \ + (tenant, entity_type, generation, vector_set) VALUES (?1, ?2, 0, '') \ + ON CONFLICT(tenant, entity_type) DO NOTHING", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + let mut rows = tx + .query( + "SELECT generation FROM entity_vector_reconciliation_generation \ + WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + let generation = rows + .next() + .await + .map_err(storage_error)? + .ok_or_else(|| { + PersistenceError::Storage(format!( + "missing vector reconciliation generation for {tenant}:{entity_type}" + )) + })? + .get::(0) + .map_err(storage_error)?; + Ok(generation as u64) +} + +async fn validate_spec_declaration_fingerprint( + tx: &libsql::Transaction, + tenant: &str, + entity_type: &str, + reconcile_vectors: bool, + spec_declaration_fingerprint: Option<&str>, +) -> Result<(), PersistenceError> { + let Some(provided_fingerprint) = spec_declaration_fingerprint else { + if reconcile_vectors { + return Err(PersistenceError::Storage(format!( + "vector reconciliation append requires a spec declaration fingerprint for {tenant}:{entity_type}" + ))); + } + return Ok(()); + }; + if provided_fingerprint.is_empty() { + return Err(PersistenceError::Storage(format!( + "live append requires a nonempty spec declaration fingerprint for {tenant}:{entity_type}" + ))); + } + + // Compatibility constructors can supply verified in-memory specs over a + // truly empty store. Establish first-writer authority atomically only when + // neither a durable catalog row nor a tombstone/authority row exists. Once + // either exists, normal catalog mutation is the sole authority. + tx.execute( + "INSERT INTO spec_declaration_authority \ + (tenant, entity_type, revision, ioa_source, declaration_fingerprint, present) \ + SELECT ?1, ?2, 1, '', ?3, 1 \ + WHERE NOT EXISTS ( \ + SELECT 1 FROM specs WHERE tenant = ?1 AND entity_type = ?2 \ + ) \ + ON CONFLICT(tenant, entity_type) DO NOTHING", + params![tenant, entity_type, provided_fingerprint], + ) + .await + .map_err(storage_error)?; + + let mut rows = tx + .query( + "SELECT ioa_source, declaration_fingerprint, present FROM spec_declaration_authority \ + WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + let authority = rows.next().await.map_err(storage_error)?.ok_or_else(|| { + PersistenceError::Storage(format!( + "missing durable spec declaration authority for {tenant}:{entity_type}" + )) + })?; + let ioa_source = authority.get::(0).map_err(storage_error)?; + let stored_fingerprint = authority.get::(1).map_err(storage_error)?; + let present = authority.get::(2).map_err(storage_error)? != 0; + drop(rows); + + let authoritative_fingerprint = if present { + if stored_fingerprint.is_empty() { + crate::spec_content_hash(&ioa_source) + } else { + stored_fingerprint + } + } else { + ABSENT_DECLARATION_FINGERPRINT.to_string() + }; + if authoritative_fingerprint != provided_fingerprint { + return Err(PersistenceError::Storage(format!( + "stale vector declaration fingerprint for {tenant}:{entity_type}" + ))); + } + Ok(()) +} + +async fn reconcile_live_vector_rows( + tx: &libsql::Transaction, + tenant: &str, + entity_type: &str, + entity_id: &str, + new_sequence: u64, + vector_rows: &[EntityVectorRow], +) -> Result<(), PersistenceError> { + let generation = current_vector_generation(tx, tenant, entity_type).await?; + let applied = tx + .execute( + "INSERT INTO entity_vector_index_version \ + (tenant, entity_type, entity_id, reconciliation_generation, sequence_nr) \ + VALUES (?1, ?2, ?3, ?4, ?5) \ + ON CONFLICT(tenant, entity_type, entity_id) DO UPDATE SET \ + reconciliation_generation = excluded.reconciliation_generation, \ + sequence_nr = excluded.sequence_nr \ + WHERE entity_vector_index_version.reconciliation_generation < excluded.reconciliation_generation \ + OR (entity_vector_index_version.reconciliation_generation = excluded.reconciliation_generation \ + AND entity_vector_index_version.sequence_nr <= excluded.sequence_nr)", + params![ + tenant, + entity_type, + entity_id, + generation as i64, + new_sequence as i64 + ], + ) + .await + .map_err(storage_error)?; + if applied == 0 { + return Err(PersistenceError::Storage(format!( + "vector-index fence for {tenant}:{entity_type}:{entity_id} is ahead of live journal sequence {new_sequence} in reconciliation generation {generation}" + ))); + } + tx.execute( + "DELETE FROM entity_vector_index \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + for row in vector_rows { + tx.execute( + "INSERT INTO entity_vector_index \ + (tenant, entity_type, decl_name, model_tag, entity_id, vector, sequence_nr) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + tenant, + entity_type, + row.decl_name.as_str(), + row.model_tag.as_str(), + entity_id, + Value::Blob(pack_f32_le(&row.vector)), + new_sequence as i64, + ], + ) + .await + .map_err(storage_error)?; + } + Ok(()) +} + impl EventStore for TursoEventStore { #[instrument(skip_all, fields(persistence_id, otel.name = "turso.append"))] async fn append( @@ -38,77 +210,8 @@ impl EventStore for TursoEventStore { expected_sequence: u64, events: &[PersistenceEnvelope], ) -> Result { - if events.is_empty() { - return Ok(expected_sequence); - } - - // Retry transient Hrana BLOCKED / stream errors with backoff (ADR-0056). - // Each attempt is a complete append unit. Single-event appends use an - // atomic conditional insert; multi-event appends open a transaction. - // Event-store's UNIQUE (entity_type, entity_id, sequence_nr) makes - // retries safe — if a prior attempt partially committed before erroring, - // the retry's pre-check detects it as ConcurrencyViolation - // (non-transient, propagates to caller via normal event-store contract). - let attempt_timeout = append_attempt_timeout(); - let total_attempts = append_max_attempts(); - let mut last_err: Option = None; - let bypass_write_gate = events.len() == 1; - for attempt in 0..total_attempts { - if attempt > 0 { - tokio::time::sleep(Duration::from_millis(retry_delay_ms(attempt - 1))).await; - } - let _high_priority_marker = if bypass_write_gate { - Some(self.mark_high_priority_write("turso.append")) - } else { - None - }; - let _write_permit = if bypass_write_gate { - None - } else { - Some( - self.acquire_write_permit("turso.append", WritePriority::High) - .await?, - ) - }; - let attempt_result = tokio::time::timeout( - attempt_timeout, - self.append_inner(persistence_id, expected_sequence, events), - ) + self.append_retried(persistence_id, expected_sequence, events, None, None) .await - .unwrap_or_else(|_| { - warn!( - persistence_id, - attempt, - timeout_ms = attempt_timeout.as_millis() as u64, - "turso.append attempt timed out" - ); - Err(PersistenceError::Storage(format!( - "turso.append timed out after {}ms", - attempt_timeout.as_millis() - ))) - }); - - match attempt_result { - Ok(seq) => { - if attempt > 0 { - record_turso_write_retry("turso.append", attempt as u64, "succeeded"); - } - return Ok(seq); - } - Err(err) => { - let transient = match &err { - PersistenceError::Storage(msg) => is_transient_write_error(msg), - _ => false, - }; - if !transient { - return Err(err); - } - last_err = Some(err); - } - } - } - record_turso_write_retry("turso.append", total_attempts as u64, "exhausted"); - Err(last_err.expect("retry loop captured at least one error")) } async fn lookup_by_key( @@ -148,12 +251,10 @@ impl EventStore for TursoEventStore { // DST. Giving Turso the keyed oracle requires first implementing live co-commit // (completing ADR-0153 phase 2 for Turso) — tracked separately. - // ADR-0155: Turso maintains `entity_vector_index` **write-behind** — the event is - // appended first (with retries), then the derived vector rows follow in a separate, - // also-retried write. This is safe for vectors (unlike keys) because a vector row - // carries no uniqueness constraint and a lagging index write only makes a ranking - // temporarily incomplete; it can never corrupt a keyed absence. So Turso implements - // the full vector surface below. + // ADR-0181: Turso co-commits the journal, retained vector fence, and current + // vector rows in one immediate transaction. The single-event fast path remains + // available only to appends that neither reconcile vectors nor carry a spec + // declaration fingerprint requiring transactional validation. async fn append_with_index_rows( &self, persistence_id: &str, @@ -162,53 +263,224 @@ impl EventStore for TursoEventStore { _key_rows: &[temper_runtime::persistence::EntityKeyRow], vector_rows: &[EntityVectorRow], reconcile_vectors: bool, + spec_declaration_fingerprint: Option<&str>, + ) -> Result { + if !reconcile_vectors && spec_declaration_fingerprint.is_none() { + return self.append(persistence_id, expected_sequence, events).await; + } + let vector_rows = reconcile_vectors.then_some(vector_rows); + self.append_retried( + persistence_id, + expected_sequence, + events, + vector_rows, + spec_declaration_fingerprint, + ) + .await + } + + async fn begin_vector_index_reconciliation( + &self, + tenant: &str, + entity_type: &str, + vector_set: &str, + declaration_revision: u64, + declaration_fingerprint: &str, ) -> Result { - // The journal append is the durable event (keys are not maintained on Turso, - // per the note above). - let new_seq = self - .append(persistence_id, expected_sequence, events) + if declaration_revision == 0 || declaration_fingerprint.is_empty() { + return Err(PersistenceError::Storage(format!( + "vector declaration revision must be nonzero and fingerprinted for {tenant}:{entity_type}" + ))); + } + let _write_permit = self + .acquire_write_permit( + "turso.begin_vector_index_reconciliation", + WritePriority::Low, + ) .await?; - // Write-behind vector maintenance: reconcile the entity's rows (delete stale, - // insert current — an empty `vector_rows` purges a deleted/cleared entity), - // RETRIED like the event append rather than a warn-once one-shot, so a - // transient failure does not silently drop the write. On final exhaustion the - // error is logged loudly; the partition then lags until the next backfill - // reconcile runs. Only runs when the type declares vector paths. - if reconcile_vectors - && let Ok((tenant, entity_type, entity_id)) = parse_persistence_id_parts(persistence_id) - { - let total_attempts = append_max_attempts(); - let mut last_err: Option = None; - for attempt in 0..total_attempts { - if attempt > 0 { - tokio::time::sleep(Duration::from_millis(retry_delay_ms(attempt - 1))).await; - } - match self - .backfill_entity_vectors(tenant, entity_type, entity_id, vector_rows) - .await - { - Ok(()) => { - last_err = None; - break; - } - Err(err) => { - let transient = matches!(&err, PersistenceError::Storage(msg) if is_transient_write_error(msg)); - last_err = Some(err); - if !transient { - break; - } - } - } + let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + + validate_spec_declaration_fingerprint( + &tx, + tenant, + entity_type, + true, + Some(declaration_fingerprint), + ) + .await?; + + // The authority row survives hard spec deletion. Spec triggers advance it + // and fence existing vector work within the same immediate transaction. + let mut authority_rows = tx + .query( + "SELECT revision, ioa_source, declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + let authority = authority_rows + .next() + .await + .map_err(storage_error)? + .ok_or_else(|| { + PersistenceError::Storage(format!( + "missing durable spec declaration authority for {tenant}:{entity_type}" + )) + })?; + let authoritative_revision = authority.get::(0).map_err(storage_error)?; + let ioa_source = authority.get::(1).map_err(storage_error)?; + let authority_fingerprint = authority.get::(2).map_err(storage_error)?; + let present = authority.get::(3).map_err(storage_error)? != 0; + drop(authority_rows); + let stored_fingerprint = if present { + if authority_fingerprint.is_empty() { + crate::spec_content_hash(&ioa_source) + } else { + authority_fingerprint } - if let Some(error) = last_err { - error!( - persistence_id, - error = %error, - "turso vector-index write-behind failed after retries; partition lags until the next backfill reconcile" - ); + } else { + ABSENT_DECLARATION_FINGERPRINT.to_string() + }; + if stored_fingerprint != declaration_fingerprint { + return Err(PersistenceError::Storage(format!( + "stale vector declaration fingerprint for {tenant}:{entity_type}" + ))); + } + let authoritative_revision = u64::try_from(authoritative_revision).map_err(|_| { + PersistenceError::Storage(format!( + "invalid durable spec revision for {tenant}:{entity_type}" + )) + })?; + let stored_revision = i64::try_from(authoritative_revision).map_err(|_| { + PersistenceError::Storage(format!( + "vector declaration revision exhausted for {tenant}:{entity_type}" + )) + })?; + + let inserted = tx + .execute( + "INSERT INTO entity_vector_reconciliation_generation \ + (tenant, entity_type, generation, declaration_revision, declaration_fingerprint, vector_set) \ + VALUES (?1, ?2, 1, ?3, ?4, ?5) \ + ON CONFLICT(tenant, entity_type) DO NOTHING", + params![ + tenant, + entity_type, + stored_revision, + declaration_fingerprint, + vector_set + ], + ) + .await + .map_err(storage_error)?; + if inserted == 1 { + tx.execute( + "DELETE FROM vector_index_backfill_watermark \ + WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + tx.commit().await.map_err(storage_error)?; + return Ok(1); + } + + let mut current_rows = tx + .query( + "SELECT generation, declaration_revision, declaration_fingerprint, vector_set \ + FROM entity_vector_reconciliation_generation \ + WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + let current = current_rows + .next() + .await + .map_err(storage_error)? + .ok_or_else(|| { + PersistenceError::Storage(format!( + "missing vector reconciliation generation for {tenant}:{entity_type}" + )) + })?; + let generation = current.get::(0).map_err(storage_error)?; + let current_revision = current.get::(1).map_err(storage_error)?; + let current_fingerprint = current.get::(2).map_err(storage_error)?; + let current_set = current.get::(3).map_err(storage_error)?; + drop(current_rows); + + let current_revision = u64::try_from(current_revision).map_err(|_| { + PersistenceError::Storage(format!( + "invalid vector declaration revision for {tenant}:{entity_type}" + )) + })?; + if authoritative_revision < current_revision { + return Err(PersistenceError::Storage(format!( + "vector reconciliation revision {current_revision} exceeds declaration authority {authoritative_revision} for {tenant}:{entity_type}" + ))); + } + if authoritative_revision == current_revision { + if current_fingerprint == declaration_fingerprint && current_set == vector_set { + tx.commit().await.map_err(storage_error)?; + return u64::try_from(generation).map_err(|_| { + PersistenceError::Storage(format!( + "invalid vector reconciliation generation for {tenant}:{entity_type}" + )) + }); + } + if !current_fingerprint.is_empty() || !current_set.is_empty() { + return Err(PersistenceError::Storage(format!( + "conflicting vector declaration at revision {authoritative_revision} for {tenant}:{entity_type}" + ))); } } - Ok(new_seq) + + let next_generation = if authoritative_revision == current_revision { + generation + } else { + generation.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "vector reconciliation generation exhausted for {tenant}:{entity_type}" + )) + })? + }; + tx.execute( + "UPDATE entity_vector_reconciliation_generation \ + SET generation = ?3, declaration_revision = ?4, \ + declaration_fingerprint = ?5, vector_set = ?6 \ + WHERE tenant = ?1 AND entity_type = ?2", + params![ + tenant, + entity_type, + next_generation, + stored_revision, + declaration_fingerprint, + vector_set + ], + ) + .await + .map_err(storage_error)?; + // Claiming a trigger-advanced or upgraded revision withdraws any legacy + // completion claim. An exact retry returned above leaves it intact. + tx.execute( + "DELETE FROM vector_index_backfill_watermark \ + WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + tx.commit().await.map_err(storage_error)?; + u64::try_from(next_generation).map_err(|_| { + PersistenceError::Storage(format!( + "invalid vector reconciliation generation for {tenant}:{entity_type}" + )) + }) } async fn backfill_entity_vectors( @@ -216,11 +488,16 @@ impl EventStore for TursoEventStore { tenant: &str, entity_type: &str, entity_id: &str, + reconciliation_generation: u64, + observed_sequence: u64, vector_rows: &[EntityVectorRow], ) -> Result<(), PersistenceError> { - // Reconcile: DELETE all of the entity's rows, then insert the current ones. - // Empty `vector_rows` purges the entity (deleted / un-embedded). Always runs - // the delete so a purge is honored. + if reconciliation_generation == 0 { + return Err(PersistenceError::Storage( + "vector reconciliation generation zero is reserved for pre-reconciliation live writes" + .to_string(), + )); + } let _write_permit = self .acquire_write_permit("turso.backfill_entity_vectors", WritePriority::Low) .await?; @@ -229,6 +506,60 @@ impl EventStore for TursoEventStore { .transaction_with_behavior(TransactionBehavior::Immediate) .await .map_err(storage_error)?; + let current_generation = current_vector_generation(&tx, tenant, entity_type).await?; + if current_generation != reconciliation_generation { + let _ = tx.rollback().await; + return Err(PersistenceError::Storage(format!( + "stale vector reconciliation generation {reconciliation_generation} for {tenant}:{entity_type}; current generation is {current_generation}" + ))); + } + let applied = tx + .execute( + "INSERT INTO entity_vector_index_version \ + (tenant, entity_type, entity_id, reconciliation_generation, sequence_nr) \ + VALUES (?1, ?2, ?3, ?4, ?5) \ + ON CONFLICT(tenant, entity_type, entity_id) DO UPDATE SET \ + reconciliation_generation = excluded.reconciliation_generation, \ + sequence_nr = excluded.sequence_nr \ + WHERE entity_vector_index_version.reconciliation_generation < excluded.reconciliation_generation \ + OR (entity_vector_index_version.reconciliation_generation = excluded.reconciliation_generation \ + AND entity_vector_index_version.sequence_nr <= excluded.sequence_nr)", + params![ + tenant, + entity_type, + entity_id, + reconciliation_generation as i64, + observed_sequence as i64 + ], + ) + .await + .map_err(storage_error)?; + if applied == 0 { + let mut rows = tx + .query( + "SELECT reconciliation_generation FROM entity_vector_index_version \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + let fence_generation = rows + .next() + .await + .map_err(storage_error)? + .map(|row| row.get::(0).map_err(storage_error)) + .transpose()? + .unwrap_or(0) as u64; + drop(rows); + if fence_generation > reconciliation_generation { + let _ = tx.rollback().await; + return Err(PersistenceError::Storage(format!( + "vector-index fence generation {fence_generation} is ahead of current type generation {reconciliation_generation} for {tenant}:{entity_type}:{entity_id}" + ))); + } + tx.commit().await.map_err(storage_error)?; + return Ok(()); + } tx.execute( "DELETE FROM entity_vector_index \ WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", @@ -240,7 +571,7 @@ impl EventStore for TursoEventStore { tx.execute( "INSERT INTO entity_vector_index \ (tenant, entity_type, decl_name, model_tag, entity_id, vector, sequence_nr) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, 0)", + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", params![ tenant, entity_type, @@ -248,6 +579,7 @@ impl EventStore for TursoEventStore { row.model_tag.as_str(), entity_id, Value::Blob(pack_f32_le(&row.vector)), + observed_sequence as i64, ], ) .await @@ -290,14 +622,54 @@ impl EventStore for TursoEventStore { &self, tenant: &str, entity_type: &str, + reconciliation_generation: u64, vector_set: &str, ) -> Result<(), PersistenceError> { + if reconciliation_generation == 0 { + return Err(PersistenceError::Storage( + "vector reconciliation generation zero cannot publish a watermark".to_string(), + )); + } let _write_permit = self .acquire_write_permit("turso.mark_vector_index_backfilled", WritePriority::Low) .await?; let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + let mut generations = tx + .query( + "SELECT generation, vector_set FROM entity_vector_reconciliation_generation \ + WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + let current = generations + .next() + .await + .map_err(storage_error)? + .map(|row| { + Ok::<_, PersistenceError>(( + row.get::(0).map_err(storage_error)? as u64, + row.get::(1).map_err(storage_error)?, + )) + }) + .transpose()?; + drop(generations); + if current.as_ref().map(|(generation, signature)| { + *generation == reconciliation_generation && signature == vector_set + }) != Some(true) + { + let current_generation = current.map(|(generation, _)| generation).unwrap_or(0); + let _ = tx.rollback().await; + return Err(PersistenceError::Storage(format!( + "stale vector reconciliation generation {reconciliation_generation} for {tenant}:{entity_type}; current generation is {current_generation}" + ))); + } let completed_at = temper_runtime::scheduler::sim_now().to_rfc3339(); - conn.execute( + tx.execute( "INSERT INTO vector_index_backfill_watermark (tenant, entity_type, vector_set, completed_at) \ VALUES (?1, ?2, ?3, ?4) \ ON CONFLICT(tenant, entity_type) \ @@ -306,6 +678,7 @@ impl EventStore for TursoEventStore { ) .await .map_err(storage_error)?; + tx.commit().await.map_err(storage_error)?; Ok(()) } @@ -331,6 +704,28 @@ impl EventStore for TursoEventStore { Ok(out) } + async fn vector_reconciliation_entity_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT entity_type FROM entity_vector_reconciliation_generation WHERE tenant = ?1 \ + UNION SELECT entity_type FROM entity_vector_index_version WHERE tenant = ?1 \ + UNION SELECT entity_type FROM entity_vector_index WHERE tenant = ?1 \ + ORDER BY entity_type", + params![tenant], + ) + .await + .map_err(storage_error)?; + let mut out = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + out.push(row.get::(0).map_err(storage_error)?); + } + Ok(out) + } + async fn vectored_entity_ids_for_type( &self, tenant: &str, @@ -352,6 +747,28 @@ impl EventStore for TursoEventStore { Ok(out) } + async fn list_vector_repair_entity_ids( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT DISTINCT entity_id FROM events \ + WHERE tenant = ?1 AND entity_type = ?2 \ + ORDER BY entity_id", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + let mut out = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + out.push(row.get::(0).map_err(storage_error)?); + } + Ok(out) + } + #[instrument(skip_all, fields(otel.name = "turso.append_batch"))] async fn append_batch( &self, @@ -362,10 +779,14 @@ impl EventStore for TursoEventStore { } if let [append] = appends { let sequence_nr = self - .append( + .append_with_index_rows( &append.persistence_id, append.expected_sequence, &append.events, + &[], + &append.vector_rows, + append.reconcile_vectors, + append.spec_declaration_fingerprint.as_deref(), ) .await?; return Ok(vec![PersistenceAppendResult { @@ -740,6 +1161,89 @@ impl EventStore for TursoEventStore { } impl TursoEventStore { + /// Retry one complete journal append, optionally including vector-index + /// reconciliation in the same transaction (ADR-0181). + async fn append_retried( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + vector_rows: Option<&[EntityVectorRow]>, + spec_declaration_fingerprint: Option<&str>, + ) -> Result { + if events.is_empty() && vector_rows.is_none() && spec_declaration_fingerprint.is_none() { + return Ok(expected_sequence); + } + + let attempt_timeout = append_attempt_timeout(); + let total_attempts = append_max_attempts(); + let mut last_err: Option = None; + let bypass_write_gate = + events.len() == 1 && vector_rows.is_none() && spec_declaration_fingerprint.is_none(); + for attempt in 0..total_attempts { + if attempt > 0 { + tokio::time::sleep(Duration::from_millis(retry_delay_ms(attempt - 1))).await; + } + let _high_priority_marker = if bypass_write_gate { + Some(self.mark_high_priority_write("turso.append")) + } else { + None + }; + let _write_permit = if bypass_write_gate { + None + } else { + Some( + self.acquire_write_permit("turso.append", WritePriority::High) + .await?, + ) + }; + let attempt_result = tokio::time::timeout( + attempt_timeout, + self.append_inner( + persistence_id, + expected_sequence, + events, + vector_rows, + spec_declaration_fingerprint, + ), + ) + .await + .unwrap_or_else(|_| { + warn!( + persistence_id, + attempt, + timeout_ms = attempt_timeout.as_millis() as u64, + "turso.append attempt timed out" + ); + Err(PersistenceError::Storage(format!( + "turso.append timed out after {}ms", + attempt_timeout.as_millis() + ))) + }); + + match attempt_result { + Ok(sequence_nr) => { + if attempt > 0 { + record_turso_write_retry("turso.append", attempt as u64, "succeeded"); + } + return Ok(sequence_nr); + } + Err(err) => { + let transient = match &err { + PersistenceError::Storage(message) => is_transient_write_error(message), + _ => false, + }; + if !transient { + return Err(err); + } + last_err = Some(err); + } + } + } + record_turso_write_retry("turso.append", total_attempts as u64, "exhausted"); + Err(last_err.expect("retry loop captured at least one error")) + } + /// List tenants with at least one persisted event. #[instrument(skip_all, fields(otel.name = "turso.list_event_tenants"))] pub async fn list_event_tenants(&self) -> Result, PersistenceError> { @@ -811,12 +1315,17 @@ impl TursoEventStore { persistence_id: &str, expected_sequence: u64, events: &[PersistenceEnvelope], + vector_rows: Option<&[EntityVectorRow]>, + spec_declaration_fingerprint: Option<&str>, ) -> Result { - if events.is_empty() { + if events.is_empty() && vector_rows.is_none() && spec_declaration_fingerprint.is_none() { return Ok(expected_sequence); } - if let [event] = events { + if vector_rows.is_none() + && spec_declaration_fingerprint.is_none() + && let [event] = events + { return self .append_single_event_inner(persistence_id, expected_sequence, event) .await; @@ -830,6 +1339,19 @@ impl TursoEventStore { .await .map_err(storage_error)?; + validate_spec_declaration_fingerprint( + &tx, + tenant, + entity_type, + vector_rows.is_some(), + spec_declaration_fingerprint, + ) + .await?; + if events.is_empty() && vector_rows.is_none() { + tx.commit().await.map_err(storage_error)?; + return Ok(expected_sequence); + } + let select_start = std::time::Instant::now(); let rows_result = tx .query( @@ -983,6 +1505,11 @@ impl TursoEventStore { .map_err(storage_error)?; } + if let Some(vector_rows) = vector_rows { + reconcile_live_vector_rows(&tx, tenant, entity_type, entity_id, new_seq, vector_rows) + .await?; + } + tx.commit().await.map_err(storage_error)?; Ok(new_seq) } @@ -1013,6 +1540,15 @@ impl TursoEventStore { parse_persistence_id_parts(&append.persistence_id) .map_err(PersistenceError::Storage)?; + validate_spec_declaration_fingerprint( + &tx, + tenant, + entity_type, + append.reconcile_vectors, + append.spec_declaration_fingerprint.as_deref(), + ) + .await?; + if append.expected_sequence == 0 && !append.events.is_empty() { parsed.push(( tenant.to_string(), @@ -1148,6 +1684,22 @@ impl TursoEventStore { } } + for ((append, result), (tenant, entity_type, entity_id)) in + appends.iter().zip(results.iter()).zip(parsed.iter()) + { + if append.reconcile_vectors { + reconcile_live_vector_rows( + &tx, + tenant, + entity_type, + entity_id, + result.sequence_nr, + &append.vector_rows, + ) + .await?; + } + } + tx.commit().await.map_err(storage_error)?; Ok(results) } diff --git a/crates/temper-store-turso/src/store/mod.rs b/crates/temper-store-turso/src/store/mod.rs index fba0d839d..39a747bcb 100644 --- a/crates/temper-store-turso/src/store/mod.rs +++ b/crates/temper-store-turso/src/store/mod.rs @@ -32,6 +32,7 @@ mod policy; mod published_artifacts; mod query_page; mod secrets; +mod spec_publication; mod specs; #[cfg(test)] mod tests; @@ -152,6 +153,9 @@ impl TursoEventStore { conn.execute(schema::CREATE_SPECS_TABLE, ()) .await .map_err(storage_error)?; + conn.execute(schema::CREATE_STAGED_SPECS_TABLE, ()) + .await + .map_err(storage_error)?; conn.execute(schema::CREATE_TRAJECTORIES_TABLE, ()) .await .map_err(storage_error)?; @@ -287,6 +291,22 @@ impl TursoEventStore { // Specs table extensions — add content_hash column for verification caching. let _ = conn.execute(schema::ALTER_SPECS_ADD_CONTENT_HASH, ()).await; let _ = conn.execute(schema::ALTER_SPECS_ADD_COMMITTED, ()).await; + conn.execute( + "INSERT INTO staged_specs \ + (tenant, entity_type, ioa_source, csdl_xml, content_hash, version, updated_at) \ + SELECT tenant, entity_type, ioa_source, csdl_xml, content_hash, version, updated_at \ + FROM specs WHERE committed = 0 \ + ON CONFLICT (tenant, entity_type) DO UPDATE SET \ + ioa_source = excluded.ioa_source, csdl_xml = excluded.csdl_xml, \ + content_hash = excluded.content_hash, version = excluded.version, \ + updated_at = excluded.updated_at", + (), + ) + .await + .map_err(storage_error)?; + conn.execute("DELETE FROM specs WHERE committed = 0", ()) + .await + .map_err(storage_error)?; // Trajectory table extensions — ALTER TABLE to add missing columns. // SQLite returns an error for duplicate columns, so we ignore failures. @@ -391,8 +411,8 @@ impl TursoEventStore { .await .map_err(storage_error)?; - // Entity vector index (ADR-0155) — declared vector paths for exact-scan kNN, - // maintained write-behind (the event append is followed by the index write). + // Entity vector index (ADR-0155/ADR-0181) — declared vector paths for + // exact-scan kNN, co-committed with a retained per-entity sequence fence. conn.execute(schema::CREATE_ENTITY_VECTOR_INDEX_TABLE, ()) .await .map_err(storage_error)?; @@ -402,9 +422,70 @@ impl TursoEventStore { conn.execute(schema::CREATE_ENTITY_VECTOR_INDEX_ENTITY, ()) .await .map_err(storage_error)?; + conn.execute(schema::CREATE_ENTITY_VECTOR_INDEX_VERSION_TABLE, ()) + .await + .map_err(storage_error)?; + if let Err(error) = conn + .execute(schema::ALTER_ENTITY_VECTOR_INDEX_VERSION_ADD_GENERATION, ()) + .await + { + let message = error.to_string(); + if !message.contains("duplicate column name") { + return Err(storage_error(error)); + } + } + conn.execute(schema::CREATE_VECTOR_RECONCILIATION_GENERATION_TABLE, ()) + .await + .map_err(storage_error)?; + for statement in [ + schema::query_plane::ALTER_VECTOR_RECONCILIATION_ADD_DECLARATION_REVISION, + schema::query_plane::ALTER_VECTOR_RECONCILIATION_ADD_DECLARATION_FINGERPRINT, + ] { + if let Err(error) = conn.execute(statement, ()).await { + let message = error.to_string(); + if !message.contains("duplicate column name") { + return Err(storage_error(error)); + } + } + } + conn.execute(schema::SEED_ENTITY_VECTOR_INDEX_VERSION_TABLE, ()) + .await + .map_err(storage_error)?; conn.execute(schema::CREATE_VECTOR_INDEX_BACKFILL_WATERMARK, ()) .await .map_err(storage_error)?; + conn.execute( + schema::declaration_authority::CREATE_SPEC_DECLARATION_AUTHORITY_TABLE, + (), + ) + .await + .map_err(storage_error)?; + if let Err(error) = conn + .execute( + schema::declaration_authority::ALTER_SPEC_DECLARATION_AUTHORITY_ADD_FINGERPRINT, + (), + ) + .await + { + let message = error.to_string(); + if !message.contains("duplicate column name") { + return Err(storage_error(error)); + } + } + // DDL is transactional in SQLite/libSQL. Holding the immediate write + // transaction across seed + trigger replacement prevents another replica + // from mutating `specs` in a drop/create gap during concurrent startup. + let authority_tx = conn + .transaction_with_behavior(libsql::TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + for statement in schema::declaration_authority::DECLARATION_AUTHORITY_STATEMENTS { + authority_tx + .execute(statement, ()) + .await + .map_err(storage_error)?; + } + authority_tx.commit().await.map_err(storage_error)?; Ok(()) } diff --git a/crates/temper-store-turso/src/store/spec_publication.rs b/crates/temper-store-turso/src/store/spec_publication.rs new file mode 100644 index 000000000..e269a978a --- /dev/null +++ b/crates/temper-store-turso/src/store/spec_publication.rs @@ -0,0 +1,246 @@ +//! Atomic publication of staged Turso specs. + +use libsql::{TransactionBehavior, params}; +use temper_runtime::persistence::{PersistenceError, storage_error}; +use tracing::instrument; + +use super::{TursoEventStore, write_gate::WritePriority}; +use crate::TursoSpecVerificationUpdate; +use crate::metrics::TursoQueryTimer; + +impl TursoEventStore { + /// Mark all uncommitted specs for a tenant as committed. + #[instrument(skip_all, fields(tenant, otel.name = "turso.commit_specs"))] + pub async fn commit_specs(&self, tenant: &str) -> Result<(), PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.commit_specs"); + let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + tx.execute( + "INSERT INTO specs ( + tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, + verified, verification_status, updated_at + ) + SELECT tenant, entity_type, ioa_source, csdl_xml, content_hash, 1, version, + 0, 'pending', datetime('now') + FROM staged_specs WHERE tenant = ?1 + ON CONFLICT (tenant, entity_type) DO UPDATE SET + ioa_source = excluded.ioa_source, + csdl_xml = excluded.csdl_xml, + content_hash = excluded.content_hash, + committed = 1, + version = specs.version + 1, + verified = 0, + verification_status = 'pending', + levels_passed = NULL, + levels_total = NULL, + verification_result = NULL, + updated_at = datetime('now')", + params![tenant], + ) + .await + .map_err(storage_error)?; + tx.execute( + "DELETE FROM staged_specs WHERE tenant = ?1", + params![tenant], + ) + .await + .map_err(storage_error)?; + tx.commit().await.map_err(storage_error)?; + Ok(()) + } + + /// Atomically promote only staged specs matching one operation's exact bytes. + #[instrument(skip_all, fields(tenant, otel.name = "turso.commit_spec_batch"))] + pub async fn commit_spec_batch( + &self, + tenant: &str, + expected: &[(&str, &str, &str)], + ) -> Result<(), PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.commit_spec_batch"); + let mut expected = expected.to_vec(); + expected.sort_unstable_by(|left, right| left.0.cmp(right.0)); + if expected.windows(2).any(|pair| pair[0].0 == pair[1].0) { + return Err(PersistenceError::Storage(format!( + "duplicate spec batch entity type for tenant {tenant}" + ))); + } + let _write_permit = self + .acquire_write_permit("turso.commit_spec_batch", WritePriority::High) + .await?; + let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + for (entity_type, content_hash, csdl_xml) in expected { + let promoted = tx + .execute( + "INSERT INTO specs ( + tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, + verified, verification_status, updated_at + ) + SELECT tenant, entity_type, ioa_source, csdl_xml, content_hash, 1, version, + 0, 'pending', datetime('now') + FROM staged_specs + WHERE tenant = ?1 AND entity_type = ?2 AND content_hash = ?3 + AND csdl_xml IS ?4 + ON CONFLICT (tenant, entity_type) DO UPDATE SET + ioa_source = excluded.ioa_source, + csdl_xml = excluded.csdl_xml, + content_hash = excluded.content_hash, + committed = 1, + version = specs.version + 1, + verified = 0, + verification_status = 'pending', + levels_passed = NULL, + levels_total = NULL, + verification_result = NULL, + updated_at = datetime('now')", + params![tenant, entity_type, content_hash, csdl_xml], + ) + .await + .map_err(storage_error)?; + if promoted != 1 { + return Err(PersistenceError::Storage(format!( + "staged spec fingerprint changed for {tenant}/{entity_type}" + ))); + } + tx.execute( + "DELETE FROM staged_specs + WHERE tenant = ?1 AND entity_type = ?2 AND content_hash = ?3 + AND csdl_xml IS ?4", + params![tenant, entity_type, content_hash, csdl_xml], + ) + .await + .map_err(storage_error)?; + } + tx.commit().await.map_err(storage_error)?; + Ok(()) + } + + /// Atomically persist verification and commit only the expected spec bytes. + #[instrument(skip_all, fields(tenant, entity_type, otel.name = "turso.commit_verified_spec"))] + pub async fn commit_verified_spec( + &self, + tenant: &str, + entity_type: &str, + expected_content_hash: &str, + expected_csdl_xml: &str, + update: TursoSpecVerificationUpdate<'_>, + ) -> Result<(), PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.commit_verified_spec"); + let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + let staged = { + let mut rows = tx + .query( + "SELECT ioa_source, csdl_xml, content_hash, version \ + FROM staged_specs \ + WHERE tenant = ?1 AND entity_type = ?2 AND content_hash = ?3 \ + AND csdl_xml IS ?4", + params![ + tenant, + entity_type, + expected_content_hash, + expected_csdl_xml + ], + ) + .await + .map_err(storage_error)?; + rows.next() + .await + .map_err(storage_error)? + .map(|row| { + Ok::<_, PersistenceError>(( + row.get::(0).map_err(storage_error)?, + row.get::>(1).map_err(storage_error)?, + row.get::(2).map_err(storage_error)?, + row.get::(3).map_err(storage_error)?, + )) + }) + .transpose()? + }; + let Some((ioa_source, csdl_xml, content_hash, version)) = staged else { + return Err(PersistenceError::Storage(format!( + "staged spec fingerprint changed for {tenant}/{entity_type}" + ))); + }; + tx.execute( + "INSERT INTO specs ( + tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, + verified, verification_status, levels_passed, levels_total, + verification_result, updated_at + ) VALUES (?1, ?2, ?3, ?4, ?5, 1, ?6, ?7, ?8, ?9, ?10, ?11, datetime('now')) + ON CONFLICT (tenant, entity_type) DO UPDATE SET + ioa_source = excluded.ioa_source, + csdl_xml = excluded.csdl_xml, + content_hash = excluded.content_hash, + committed = 1, + version = specs.version + 1, + verified = excluded.verified, + verification_status = excluded.verification_status, + levels_passed = excluded.levels_passed, + levels_total = excluded.levels_total, + verification_result = excluded.verification_result, + updated_at = datetime('now')", + params![ + tenant, + entity_type, + ioa_source, + csdl_xml, + content_hash, + version, + update.verified as i64, + update.status, + update.levels_passed, + update.levels_total, + update.verification_result_json + ], + ) + .await + .map_err(storage_error)?; + let affected = tx + .execute( + "DELETE FROM staged_specs \ + WHERE tenant = ?1 AND entity_type = ?2 AND content_hash = ?3 \ + AND csdl_xml IS ?4", + params![ + tenant, + entity_type, + expected_content_hash, + expected_csdl_xml + ], + ) + .await + .map_err(storage_error)?; + if affected != 1 { + return Err(PersistenceError::Storage(format!( + "staged spec fingerprint changed for {tenant}/{entity_type}" + ))); + } + tx.commit().await.map_err(storage_error)?; + Ok(()) + } + + /// Delete all uncommitted specs across all tenants. + #[instrument(skip_all, fields(otel.name = "turso.delete_uncommitted_specs"))] + pub async fn delete_uncommitted_specs(&self) -> Result { + let _query_timer = TursoQueryTimer::start("turso.delete_uncommitted_specs"); + let conn = self.configured_connection().await?; + let staged = conn + .execute("DELETE FROM staged_specs", ()) + .await + .map_err(storage_error)?; + let legacy = conn + .execute("DELETE FROM specs WHERE committed = 0", ()) + .await + .map_err(storage_error)?; + Ok((staged + legacy) as usize) + } +} diff --git a/crates/temper-store-turso/src/store/specs.rs b/crates/temper-store-turso/src/store/specs.rs index 5edb0ab4e..2d506c9de 100644 --- a/crates/temper-store-turso/src/store/specs.rs +++ b/crates/temper-store-turso/src/store/specs.rs @@ -1,6 +1,6 @@ //! Spec persistence: upsert, verification updates, and startup loading. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use libsql::{TransactionBehavior, params}; use temper_runtime::persistence::{PersistenceError, storage_error}; @@ -18,6 +18,134 @@ struct ExistingSpecFingerprint { } impl TursoEventStore { + /// Atomically publish an exact verified staged catalog. + pub async fn persist_verified_spec_catalog_update( + &self, + tenant: &str, + expected: &[(&str, &str, &str)], + additional_removed_entity_types: &[String], + replace: bool, + cross_invariants_toml: Option<&str>, + ) -> Result, PersistenceError> { + let mut expected = expected.to_vec(); + expected.sort_unstable_by(|left, right| left.0.cmp(right.0)); + if expected.windows(2).any(|pair| pair[0].0 == pair[1].0) { + return Err(PersistenceError::Storage(format!( + "duplicate verified catalog entity type for tenant {tenant}" + ))); + } + let _write_permit = self + .acquire_write_permit( + "turso.persist_verified_spec_catalog_update", + WritePriority::High, + ) + .await?; + let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + let incoming = expected + .iter() + .map(|(entity_type, _, _)| *entity_type) + .collect::>(); + let mut removed_entity_types = BTreeSet::new(); + if replace { + let mut rows = tx + .query( + "SELECT entity_type FROM specs WHERE tenant = ?1 \ + UNION SELECT entity_type FROM staged_specs WHERE tenant = ?1 \ + UNION SELECT entity_type FROM spec_declaration_authority \ + WHERE tenant = ?1 AND present = 1 ORDER BY entity_type", + params![tenant], + ) + .await + .map_err(storage_error)?; + while let Some(row) = rows.next().await.map_err(storage_error)? { + let entity_type = row.get::(0).map_err(storage_error)?; + if !incoming.contains(entity_type.as_str()) { + removed_entity_types.insert(entity_type); + } + } + } + removed_entity_types.extend( + additional_removed_entity_types + .iter() + .filter(|entity_type| !incoming.contains(entity_type.as_str())) + .cloned(), + ); + let removed_entity_types = removed_entity_types.into_iter().collect::>(); + + for (entity_type, content_hash, csdl_xml) in expected { + let promoted = tx + .execute( + "INSERT INTO specs ( + tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, + verified, verification_status, updated_at + ) + SELECT tenant, entity_type, ioa_source, csdl_xml, content_hash, 1, version, + 1, 'passed', datetime('now') + FROM staged_specs + WHERE tenant = ?1 AND entity_type = ?2 AND content_hash = ?3 + AND csdl_xml IS ?4 + ON CONFLICT (tenant, entity_type) DO UPDATE SET + ioa_source = excluded.ioa_source, + csdl_xml = excluded.csdl_xml, + content_hash = excluded.content_hash, + committed = 1, + version = specs.version + 1, + verified = 1, + verification_status = 'passed', + levels_passed = NULL, + levels_total = NULL, + verification_result = NULL, + updated_at = datetime('now')", + params![tenant, entity_type, content_hash, csdl_xml], + ) + .await + .map_err(storage_error)?; + if promoted != 1 { + return Err(PersistenceError::Storage(format!( + "staged spec fingerprint changed for {tenant}/{entity_type}" + ))); + } + tx.execute( + "DELETE FROM staged_specs + WHERE tenant = ?1 AND entity_type = ?2 AND content_hash = ?3 + AND csdl_xml IS ?4", + params![tenant, entity_type, content_hash, csdl_xml], + ) + .await + .map_err(storage_error)?; + } + for entity_type in &removed_entity_types { + Self::tombstone_spec_in_transaction(&tx, tenant, entity_type).await?; + } + if let Some(source) = cross_invariants_toml { + tx.execute( + "INSERT INTO tenant_constraints + (tenant, cross_invariants_toml, version, updated_at) + VALUES (?1, ?2, 1, datetime('now')) + ON CONFLICT(tenant) DO UPDATE SET + cross_invariants_toml = excluded.cross_invariants_toml, + version = tenant_constraints.version + 1, + updated_at = datetime('now')", + params![tenant, source], + ) + .await + .map_err(storage_error)?; + } else if replace { + tx.execute( + "DELETE FROM tenant_constraints WHERE tenant = ?1", + params![tenant], + ) + .await + .map_err(storage_error)?; + } + tx.commit().await.map_err(storage_error)?; + Ok(removed_entity_types) + } + /// Upsert a spec source (IOA + CSDL) for a tenant/entity_type. /// /// Uses content-hash gating: if the spec already exists with the same @@ -37,45 +165,21 @@ impl TursoEventStore { .acquire_write_permit("turso.upsert_spec", WritePriority::High) .await?; let conn = self.configured_connection().await?; - // When content_hash matches the existing row, keep verification intact. - // Otherwise reset to pending so the cascade re-runs. + // Staging is versioned separately so an interrupted verifier cannot + // overwrite the last committed, restorable catalog row. conn.execute( - "INSERT INTO specs (tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, verified, verification_status, updated_at) - VALUES (?1, ?2, ?3, ?4, ?5, 0, 1, 0, 'pending', datetime('now')) + "INSERT INTO staged_specs (tenant, entity_type, ioa_source, csdl_xml, content_hash, version, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, 1, datetime('now')) ON CONFLICT (tenant, entity_type) DO UPDATE SET - ioa_source = CASE - WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml - THEN excluded.ioa_source ELSE specs.ioa_source END, - csdl_xml = CASE - WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml - THEN excluded.csdl_xml ELSE specs.csdl_xml END, - content_hash = CASE - WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml - THEN excluded.content_hash ELSE specs.content_hash END, - committed = CASE - WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml - THEN 0 ELSE specs.committed END, + ioa_source = excluded.ioa_source, + csdl_xml = excluded.csdl_xml, + content_hash = excluded.content_hash, version = CASE - WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml - THEN specs.version + 1 ELSE specs.version END, - verified = CASE - WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml - THEN 0 ELSE specs.verified END, - verification_status = CASE - WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml - THEN 'pending' ELSE specs.verification_status END, - levels_passed = CASE - WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml - THEN NULL ELSE specs.levels_passed END, - levels_total = CASE - WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml - THEN NULL ELSE specs.levels_total END, - verification_result = CASE - WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml - THEN NULL ELSE specs.verification_result END, + WHEN staged_specs.content_hash IS NOT excluded.content_hash OR staged_specs.csdl_xml IS NOT excluded.csdl_xml + THEN staged_specs.version + 1 ELSE staged_specs.version END, updated_at = CASE - WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml - THEN datetime('now') ELSE specs.updated_at END", + WHEN staged_specs.content_hash IS NOT excluded.content_hash OR staged_specs.csdl_xml IS NOT excluded.csdl_xml + THEN datetime('now') ELSE staged_specs.updated_at END", params![tenant, entity_type, ioa_source, csdl_xml, content_hash], ) .await @@ -83,6 +187,123 @@ impl TursoEventStore { Ok(()) } + /// Atomically publish one hot-loaded spec catalog update. + /// + /// When `replace` is true, omissions are discovered only after the Immediate + /// transaction owns the database write lock. Replacement omissions are + /// tombstoned with the supplied committed specs and tenant constraints, so + /// concurrent replicas cannot leave a union that neither source advertised. + /// An omitted constraint source is preserved for merges and cleared for + /// replacements. + pub async fn persist_spec_catalog_update( + &self, + tenant: &str, + specs: &[(&str, &str, &str)], + csdl_xml: &str, + additional_removed_entity_types: &[String], + replace: bool, + cross_invariants_toml: Option<&str>, + ) -> Result, PersistenceError> { + let _write_permit = self + .acquire_write_permit("turso.persist_spec_catalog_update", WritePriority::High) + .await?; + let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + let incoming = specs + .iter() + .map(|(entity_type, _, _)| *entity_type) + .collect::>(); + let mut removed_entity_types = if replace { + let mut rows = tx + .query( + "SELECT entity_type FROM specs WHERE tenant = ?1 \ + UNION \ + SELECT entity_type FROM staged_specs WHERE tenant = ?1 \ + UNION \ + SELECT entity_type FROM spec_declaration_authority \ + WHERE tenant = ?1 AND present = 1 \ + ORDER BY entity_type", + params![tenant], + ) + .await + .map_err(storage_error)?; + let mut removed = BTreeSet::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + let entity_type = row.get::(0).map_err(storage_error)?; + if !incoming.contains(entity_type.as_str()) { + removed.insert(entity_type); + } + } + removed + } else { + BTreeSet::new() + }; + removed_entity_types.extend( + additional_removed_entity_types + .iter() + .filter(|entity_type| !incoming.contains(entity_type.as_str())) + .cloned(), + ); + let removed_entity_types = removed_entity_types.into_iter().collect::>(); + + for (entity_type, ioa_source, content_hash) in specs { + tx.execute( + "DELETE FROM staged_specs WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + tx.execute( + "INSERT INTO specs \ + (tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, verified, verification_status, updated_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, 1, 1, 0, 'pending', datetime('now')) \ + ON CONFLICT (tenant, entity_type) DO UPDATE SET \ + ioa_source = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN excluded.ioa_source ELSE specs.ioa_source END, \ + csdl_xml = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN excluded.csdl_xml ELSE specs.csdl_xml END, \ + content_hash = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN excluded.content_hash ELSE specs.content_hash END, \ + committed = 1, \ + version = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN specs.version + 1 ELSE specs.version END, \ + verified = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN 0 ELSE specs.verified END, \ + verification_status = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN 'pending' ELSE specs.verification_status END, \ + levels_passed = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN NULL ELSE specs.levels_passed END, \ + levels_total = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN NULL ELSE specs.levels_total END, \ + verification_result = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN NULL ELSE specs.verification_result END, \ + updated_at = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN datetime('now') ELSE specs.updated_at END", + params![tenant, *entity_type, *ioa_source, csdl_xml, *content_hash], + ) + .await + .map_err(storage_error)?; + } + for entity_type in &removed_entity_types { + Self::tombstone_spec_in_transaction(&tx, tenant, entity_type).await?; + } + if let Some(source) = cross_invariants_toml { + tx.execute( + "INSERT INTO tenant_constraints (tenant, cross_invariants_toml, version, updated_at) \ + VALUES (?1, ?2, 1, datetime('now')) \ + ON CONFLICT(tenant) DO UPDATE SET \ + cross_invariants_toml = excluded.cross_invariants_toml, \ + version = tenant_constraints.version + 1, \ + updated_at = datetime('now')", + params![tenant, source], + ) + .await + .map_err(storage_error)?; + } else if replace { + tx.execute( + "DELETE FROM tenant_constraints WHERE tenant = ?1", + params![tenant], + ) + .await + .map_err(storage_error)?; + } + tx.commit().await.map_err(storage_error)?; + Ok(removed_entity_types) + } + /// Atomically upsert multiple specs, record the app installation, optionally /// write a Cedar policy, and mark all tenant specs as committed — all within /// a single libsql transaction. @@ -105,20 +326,38 @@ impl TursoEventStore { let policy_needs_write = Self::tenant_policy_needs_write(&conn, tenant, policy).await?; let app_needs_write = Self::installed_app_needs_write(&conn, tenant, app_name).await?; - if spec_indices.is_empty() && !policy_needs_write && !app_needs_write { + if specs.is_empty() && !policy_needs_write && !app_needs_write { return Ok(()); } - let _write_permit = self - .acquire_write_permit("turso.upsert_specs_and_commit", WritePriority::High) - .await?; + let needs_gated_write = !spec_indices.is_empty() || policy_needs_write || app_needs_write; + let _write_permit = if needs_gated_write { + Some( + self.acquire_write_permit("turso.upsert_specs_and_commit", WritePriority::High) + .await?, + ) + } else { + None + }; let tx = conn .transaction_with_behavior(TransactionBehavior::Immediate) .await .map_err(storage_error)?; - for index in spec_indices { - let (entity_type, ioa_source, csdl_xml, content_hash) = specs[index]; + // The transaction is the linearization point even when every committed + // input is byte-identical. A verifier may have staged conflicting bytes + // on another replica after the preflight reads; the authoritative app + // write supersedes every such candidate. + for (entity_type, _, _, _) in specs { + tx.execute( + "DELETE FROM staged_specs WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + } + + for (entity_type, ioa_source, csdl_xml, content_hash) in specs { tx.execute( "INSERT INTO specs (tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, verified, verification_status, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, 1, 1, 0, 'pending', datetime('now')) @@ -304,6 +543,86 @@ impl TursoEventStore { Ok(rows.next().await.map_err(storage_error)?.is_none()) } + async fn tombstone_spec_in_transaction( + tx: &libsql::Transaction, + tenant: &str, + entity_type: &str, + ) -> Result<(), PersistenceError> { + let deleted_catalog_committed = { + let mut rows = tx + .query( + "SELECT committed FROM specs WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + rows.next() + .await + .map_err(storage_error)? + .map(|row| row.get::(0).map(|committed| committed != 0)) + .transpose() + .map_err(storage_error)? + .unwrap_or(false) + }; + tx.execute( + "DELETE FROM staged_specs WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + tx.execute( + "DELETE FROM specs WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + // Only committed-row deletion fires the authority trigger. A staged + // row can hide the live declaration stored in authority, so deleting + // it must still execute the explicit tombstone below. + if deleted_catalog_committed { + return Ok(()); + } + + // Compatibility constructors may establish first-writer authority + // without a `specs` row. A later full replacement must still persist + // an absence tombstone and fence completed/in-flight vector work. + let tombstoned = tx + .execute( + "INSERT INTO spec_declaration_authority \ + (tenant, entity_type, revision, ioa_source, declaration_fingerprint, present) \ + VALUES (?1, ?2, 1, '', 'absent:v1', 0) \ + ON CONFLICT(tenant, entity_type) DO UPDATE SET \ + revision = spec_declaration_authority.revision + 1, \ + ioa_source = '', declaration_fingerprint = 'absent:v1', present = 0 \ + WHERE spec_declaration_authority.present != 0", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + if tombstoned > 0 { + tx.execute( + "UPDATE entity_vector_reconciliation_generation \ + SET generation = generation + 1, \ + declaration_revision = ( \ + SELECT revision FROM spec_declaration_authority \ + WHERE tenant = ?1 AND entity_type = ?2 \ + ), declaration_fingerprint = '', vector_set = '' \ + WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + tx.execute( + "DELETE FROM vector_index_backfill_watermark \ + WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type], + ) + .await + .map_err(storage_error)?; + } + Ok(()) + } + /// Delete a spec for a given tenant/entity_type. #[instrument(skip_all, fields(tenant, entity_type, otel.name = "turso.delete_spec"))] pub async fn delete_spec( @@ -312,13 +631,16 @@ impl TursoEventStore { entity_type: &str, ) -> Result<(), PersistenceError> { let _query_timer = TursoQueryTimer::start("turso.delete_spec"); + let _write_permit = self + .acquire_write_permit("turso.delete_spec", WritePriority::High) + .await?; let conn = self.configured_connection().await?; - conn.execute( - "DELETE FROM specs WHERE tenant = ?1 AND entity_type = ?2", - params![tenant, entity_type], - ) - .await - .map_err(storage_error)?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + Self::tombstone_spec_in_transaction(&tx, tenant, entity_type).await?; + tx.commit().await.map_err(storage_error)?; Ok(()) } @@ -592,6 +914,35 @@ impl TursoEventStore { // ── Spec Loading ────────────────────────────────────────────── + /// Entity types that a source-of-truth replacement must account for. + /// + /// Includes uncommitted catalog rows left by an interrupted load and + /// compatibility authority established without a catalog row. + pub async fn spec_replacement_entity_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT entity_type FROM specs WHERE tenant = ?1 \ + UNION \ + SELECT entity_type FROM staged_specs WHERE tenant = ?1 \ + UNION \ + SELECT entity_type FROM spec_declaration_authority \ + WHERE tenant = ?1 AND present != 0 \ + ORDER BY entity_type", + params![tenant], + ) + .await + .map_err(storage_error)?; + let mut entity_types = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + entity_types.push(row.get::(0).map_err(storage_error)?); + } + Ok(entity_types) + } + /// Load all persisted specs (for startup recovery). #[instrument(skip_all, fields(otel.name = "turso.load_specs"))] pub async fn load_specs(&self) -> Result, PersistenceError> { @@ -638,30 +989,4 @@ impl TursoEventStore { } Ok(out) } - - /// Mark all uncommitted specs for a tenant as committed. - #[instrument(skip_all, fields(tenant, otel.name = "turso.commit_specs"))] - pub async fn commit_specs(&self, tenant: &str) -> Result<(), PersistenceError> { - let _query_timer = TursoQueryTimer::start("turso.commit_specs"); - let conn = self.configured_connection().await?; - conn.execute( - "UPDATE specs SET committed = 1, updated_at = datetime('now') WHERE tenant = ?1 AND committed != 1", - params![tenant], - ) - .await - .map_err(storage_error)?; - Ok(()) - } - - /// Delete all uncommitted specs across all tenants. - #[instrument(skip_all, fields(otel.name = "turso.delete_uncommitted_specs"))] - pub async fn delete_uncommitted_specs(&self) -> Result { - let _query_timer = TursoQueryTimer::start("turso.delete_uncommitted_specs"); - let conn = self.configured_connection().await?; - let affected = conn - .execute("DELETE FROM specs WHERE committed = 0", ()) - .await - .map_err(storage_error)?; - Ok(affected as usize) - } } diff --git a/crates/temper-store-turso/src/store/tests/declaration_authority.rs b/crates/temper-store-turso/src/store/tests/declaration_authority.rs new file mode 100644 index 000000000..26c7170f4 --- /dev/null +++ b/crates/temper-store-turso/src/store/tests/declaration_authority.rs @@ -0,0 +1,911 @@ +use super::*; + +async fn item_authority(store: &TursoEventStore) -> (i64, String, i64) { + let conn = store.configured_connection().await.unwrap(); + let mut rows = conn + .query( + "SELECT revision, declaration_fingerprint, present \ + FROM spec_declaration_authority \ + WHERE tenant = 't' AND entity_type = 'Item'", + (), + ) + .await + .unwrap(); + let row = rows + .next() + .await + .unwrap() + .expect("Item declaration authority"); + ( + row.get::(0).unwrap(), + row.get::(1).unwrap(), + row.get::(2).unwrap(), + ) +} + +#[tokio::test] +async fn durable_spec_revision_rejects_stale_replica_and_allows_later_readd() { + let store = make_store("vector-durable-spec-revision").await; + let csdl = ""; + let ioa_a = "[automaton]\nname = \"ItemA\"\n"; + let ioa_b = "[automaton]\nname = \"ItemB\"\n"; + let fingerprint_a = crate::spec_content_hash(ioa_a); + let fingerprint_b = crate::spec_content_hash(ioa_b); + + store + .upsert_spec("t", "Item", ioa_a, csdl, &fingerprint_a) + .await + .unwrap(); + store.commit_specs("t").await.unwrap(); + let generation_a = store + .begin_vector_index_reconciliation("t", "Item", "v2|a", 1, &fingerprint_a) + .await + .unwrap(); + store + .mark_vector_index_backfilled("t", "Item", generation_a, "v2|a") + .await + .unwrap(); + + store + .upsert_spec("t", "Item", ioa_b, csdl, &fingerprint_b) + .await + .unwrap(); + store.commit_specs("t").await.unwrap(); + let generation_b = store + .begin_vector_index_reconciliation("t", "Item", "v2|b", 1, &fingerprint_b) + .await + .unwrap(); + store + .mark_vector_index_backfilled("t", "Item", generation_b, "v2|b") + .await + .unwrap(); + + assert!( + store + .begin_vector_index_reconciliation("t", "Item", "v2|a", 99, &fingerprint_a) + .await + .is_err(), + "a stale replica fingerprint must be rejected even with a larger caller revision" + ); + assert_eq!( + store.vector_index_backfilled_types("t").await.unwrap(), + vec![("Item".to_string(), "v2|b".to_string())] + ); + + store + .upsert_spec("t", "Item", ioa_a, csdl, &fingerprint_a) + .await + .unwrap(); + store.commit_specs("t").await.unwrap(); + let readded_a = store + .begin_vector_index_reconciliation("t", "Item", "v2|a", 1, &fingerprint_a) + .await + .unwrap(); + assert!( + readded_a > generation_b, + "a durable A re-add is a new revision" + ); +} + +#[tokio::test] +async fn staged_spec_does_not_advance_authority_until_commit() { + let store = make_store("vector-staged-declaration-authority").await; + let csdl = ""; + let ioa_a = "[automaton]\nname = \"Item\"\n# committed-a\n"; + let ioa_b = "[automaton]\nname = \"Item\"\n# staged-b\n"; + let fingerprint_a = crate::spec_content_hash(ioa_a); + let fingerprint_b = crate::spec_content_hash(ioa_b); + + store + .upsert_spec("t", "Item", ioa_a, csdl, &fingerprint_a) + .await + .unwrap(); + store.commit_specs("t").await.unwrap(); + let generation_a = store + .begin_vector_index_reconciliation("t", "Item", "v2|a", 1, &fingerprint_a) + .await + .unwrap(); + store + .mark_vector_index_backfilled("t", "Item", generation_a, "v2|a") + .await + .unwrap(); + let authority_a = item_authority(&store).await; + + store + .upsert_spec("t", "Item", ioa_b, csdl, &fingerprint_b) + .await + .unwrap(); + assert_eq!(item_authority(&store).await, authority_a); + assert_eq!( + store.vector_index_backfilled_types("t").await.unwrap(), + vec![("Item".to_string(), "v2|a".to_string())], + "uncommitted staging must not withdraw the published watermark" + ); + + assert_eq!(store.delete_uncommitted_specs().await.unwrap(), 1); + assert_eq!(item_authority(&store).await, authority_a); + assert_eq!( + store.vector_index_backfilled_types("t").await.unwrap(), + vec![("Item".to_string(), "v2|a".to_string())], + "discarding uncommitted staging must not tombstone the live declaration" + ); + + store + .upsert_spec("t", "Item", ioa_b, csdl, &fingerprint_b) + .await + .unwrap(); + store.commit_specs("t").await.unwrap(); + let authority_b = item_authority(&store).await; + assert!(authority_b.0 > authority_a.0); + assert_eq!(authority_b.1, fingerprint_b); + assert_eq!(authority_b.2, 1); + assert!( + store + .vector_index_backfilled_types("t") + .await + .unwrap() + .is_empty(), + "the false-to-true commit transition must withdraw the old watermark" + ); +} + +#[tokio::test] +async fn scoped_commit_does_not_promote_unrelated_staging() { + let store = make_store("vector-scoped-spec-commit").await; + let csdl = ""; + let item = "[automaton]\nname = \"Item\"\n"; + let unrelated = "[automaton]\nname = \"Unrelated\"\n"; + let item_fingerprint = crate::spec_content_hash(item); + let unrelated_fingerprint = crate::spec_content_hash(unrelated); + + store + .upsert_spec("t", "Item", item, csdl, &item_fingerprint) + .await + .unwrap(); + store + .upsert_spec("t", "Unrelated", unrelated, csdl, &unrelated_fingerprint) + .await + .unwrap(); + store + .commit_verified_spec( + "t", + "Item", + &item_fingerprint, + csdl, + crate::TursoSpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .unwrap(); + + let committed = store.load_specs().await.unwrap(); + assert_eq!(committed.len(), 1); + assert_eq!(committed[0].entity_type, "Item"); + let conn = store.configured_connection().await.unwrap(); + let unrelated_staged_hash: String = conn + .query( + "SELECT content_hash FROM staged_specs WHERE tenant = 't' AND entity_type = 'Unrelated'", + (), + ) + .await + .unwrap() + .next() + .await + .unwrap() + .expect("unrelated staged row") + .get(0) + .unwrap(); + assert_eq!(unrelated_staged_hash, unrelated_fingerprint); +} + +#[tokio::test] +async fn spec_batch_commit_rolls_back_every_promotion_on_mismatch() { + let store = make_store("vector-batch-spec-rollback").await; + let csdl = ""; + let item = "[automaton]\nname = \"Item\"\n"; + let issue = "[automaton]\nname = \"Issue\"\n"; + let item_hash = crate::spec_content_hash(item); + let issue_hash = crate::spec_content_hash(issue); + + store + .upsert_spec("t", "Item", item, csdl, &item_hash) + .await + .unwrap(); + store + .upsert_spec("t", "Issue", issue, csdl, &issue_hash) + .await + .unwrap(); + store + .commit_spec_batch( + "t", + &[ + ("Item", item_hash.as_str(), csdl), + ("Issue", "wrong-hash", csdl), + ], + ) + .await + .expect_err("one mismatch must roll back the whole batch"); + + assert!(store.load_specs().await.unwrap().is_empty()); + let conn = store.configured_connection().await.unwrap(); + let staged: i64 = conn + .query("SELECT COUNT(*) FROM staged_specs WHERE tenant = 't'", ()) + .await + .unwrap() + .next() + .await + .unwrap() + .expect("staged count") + .get(0) + .unwrap(); + assert_eq!(staged, 2); +} + +#[tokio::test] +async fn verified_commit_rejects_same_type_fingerprint_overwrite() { + let store = make_store("vector-same-type-verified-commit").await; + let csdl = ""; + let ioa_a = "[automaton]\nname = \"Item\"\n# verified-a\n"; + let ioa_b = "[automaton]\nname = \"Item\"\n# staged-b\n"; + let fingerprint_a = crate::spec_content_hash(ioa_a); + let fingerprint_b = crate::spec_content_hash(ioa_b); + + store + .upsert_spec("t", "Item", ioa_a, csdl, &fingerprint_a) + .await + .unwrap(); + store + .commit_verified_spec( + "t", + "Item", + &fingerprint_a, + csdl, + crate::TursoSpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .unwrap(); + store + .upsert_spec("t", "Item", ioa_b, csdl, &fingerprint_b) + .await + .unwrap(); + let error = store + .commit_verified_spec( + "t", + "Item", + &fingerprint_a, + csdl, + crate::TursoSpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .expect_err("verified A must not publish staged B"); + assert!(error.to_string().contains("fingerprint changed")); + + let conn = store.configured_connection().await.unwrap(); + let mut rows = conn + .query( + "SELECT content_hash, verified, committed FROM specs \ + WHERE tenant = 't' AND entity_type = 'Item'", + (), + ) + .await + .unwrap(); + let row = rows.next().await.unwrap().expect("committed A row"); + assert_eq!(row.get::(0).unwrap(), fingerprint_a); + assert_eq!(row.get::(1).unwrap(), 1); + assert_eq!(row.get::(2).unwrap(), 1); + let mut staged = conn + .query( + "SELECT content_hash FROM staged_specs \ + WHERE tenant = 't' AND entity_type = 'Item'", + (), + ) + .await + .unwrap(); + let staged_row = staged.next().await.unwrap().expect("staged B row"); + assert_eq!(staged_row.get::(0).unwrap(), fingerprint_b); +} + +#[tokio::test] +async fn verified_commit_rejects_same_ioa_with_replaced_csdl() { + let store = make_store("vector-same-ioa-replaced-csdl").await; + let ioa = "[automaton]\nname = \"Item\"\n"; + let fingerprint = crate::spec_content_hash(ioa); + let csdl_a = ""; + let csdl_b = ""; + + store + .upsert_spec("t", "Item", ioa, csdl_a, &fingerprint) + .await + .unwrap(); + store + .commit_verified_spec( + "t", + "Item", + &fingerprint, + csdl_a, + crate::TursoSpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .unwrap(); + store + .upsert_spec("t", "Item", ioa, csdl_b, &fingerprint) + .await + .unwrap(); + + store + .commit_verified_spec( + "t", + "Item", + &fingerprint, + csdl_a, + crate::TursoSpecVerificationUpdate { + status: "completed", + verified: true, + levels_passed: None, + levels_total: None, + verification_result_json: None, + }, + ) + .await + .expect_err("verification of CSDL A must not publish staged CSDL B"); + + let conn = store.configured_connection().await.unwrap(); + let mut committed = conn + .query( + "SELECT csdl_xml, verified FROM specs \ + WHERE tenant = 't' AND entity_type = 'Item'", + (), + ) + .await + .unwrap(); + let committed_row = committed.next().await.unwrap().expect("committed CSDL A"); + assert_eq!(committed_row.get::(0).unwrap(), csdl_a); + assert_eq!(committed_row.get::(1).unwrap(), 1); + let mut staged = conn + .query( + "SELECT csdl_xml FROM staged_specs \ + WHERE tenant = 't' AND entity_type = 'Item'", + (), + ) + .await + .unwrap(); + let staged_row = staged.next().await.unwrap().expect("staged CSDL B"); + assert_eq!(staged_row.get::(0).unwrap(), csdl_b); +} + +#[tokio::test] +async fn identical_atomic_app_write_discards_conflicting_staging() { + let store = make_store("atomic-app-write-discards-staging").await; + let ioa_a = "[automaton]\nname = \"Item\"\n# app-a\n"; + let ioa_b = "[automaton]\nname = \"Item\"\n# staged-b\n"; + let csdl = ""; + let fingerprint_a = crate::spec_content_hash(ioa_a); + let fingerprint_b = crate::spec_content_hash(ioa_b); + let app_specs = [("Item", ioa_a, csdl, fingerprint_a.as_str())]; + + store + .upsert_specs_and_commit("t", &app_specs, None, "test-app") + .await + .unwrap(); + store + .upsert_spec("t", "Item", ioa_b, csdl, &fingerprint_b) + .await + .unwrap(); + + store + .upsert_specs_and_commit("t", &app_specs, None, "test-app") + .await + .unwrap(); + + let committed = store.load_specs().await.unwrap(); + assert_eq!(committed.len(), 1); + assert_eq!( + committed[0].content_hash.as_deref(), + Some(fingerprint_a.as_str()) + ); + let conn = store.configured_connection().await.unwrap(); + let mut staged = conn + .query( + "SELECT 1 FROM staged_specs \ + WHERE tenant = 't' AND entity_type = 'Item'", + (), + ) + .await + .unwrap(); + assert!( + staged.next().await.unwrap().is_none(), + "the authoritative app write must discard staged B" + ); +} + +#[tokio::test] +async fn full_replacement_tombstones_authority_hidden_by_staged_catalog_row() { + let store = make_store("vector-staged-full-replacement-tombstone").await; + let csdl = ""; + let ioa_a = "[automaton]\nname = \"Item\"\n# committed-a\n"; + let ioa_b = "[automaton]\nname = \"Item\"\n# staged-b\n"; + let fingerprint_a = crate::spec_content_hash(ioa_a); + let fingerprint_b = crate::spec_content_hash(ioa_b); + + store + .upsert_spec("t", "Item", ioa_a, csdl, &fingerprint_a) + .await + .unwrap(); + store.commit_specs("t").await.unwrap(); + let generation_a = store + .begin_vector_index_reconciliation("t", "Item", "v2|a", 1, &fingerprint_a) + .await + .unwrap(); + store + .mark_vector_index_backfilled("t", "Item", generation_a, "v2|a") + .await + .unwrap(); + let authority_a = item_authority(&store).await; + + store + .upsert_spec("t", "Item", ioa_b, csdl, &fingerprint_b) + .await + .unwrap(); + assert_eq!(item_authority(&store).await, authority_a); + + assert_eq!( + store + .persist_spec_catalog_update("t", &[], csdl, &[], true, None) + .await + .unwrap(), + vec!["Item".to_string()] + ); + let tombstone = item_authority(&store).await; + assert!(tombstone.0 > authority_a.0); + assert_eq!(tombstone.1, "absent:v1"); + assert_eq!(tombstone.2, 0); + assert!( + store + .vector_index_backfilled_types("t") + .await + .unwrap() + .is_empty(), + "full replacement must withdraw the committed declaration even when its catalog row is staged" + ); +} + +#[tokio::test] +async fn fresh_store_atomically_bootstraps_first_fingerprinted_declaration() { + let store = make_store("vector-fresh-authority-bootstrap").await; + let fingerprint_a = crate::spec_content_hash("fresh declaration A"); + let fingerprint_b = crate::spec_content_hash("fresh declaration B"); + + let generation = store + .begin_vector_index_reconciliation("t", "Item", "v2|embed-a", 1, &fingerprint_a) + .await + .expect("first in-memory declaration should establish empty-store authority"); + store + .append_with_index_rows( + "t:Item:item-fresh", + 0, + &[test_envelope("Created", serde_json::json!({}))], + &[], + &[EntityVectorRow { + decl_name: "embed-a".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }], + true, + Some(&fingerprint_a), + ) + .await + .expect("the authoritative fresh declaration should write"); + + assert!( + store + .begin_vector_index_reconciliation("t", "Item", "v2|embed-b", 99, &fingerprint_b) + .await + .is_err(), + "a different process-local declaration must not replace first-writer authority" + ); + assert_eq!( + store + .begin_vector_index_reconciliation("t", "Item", "v2|embed-a", 1, &fingerprint_a) + .await + .unwrap(), + generation + ); + assert_eq!( + store + .read_events("t:Item:item-fresh", 0) + .await + .unwrap() + .len(), + 1 + ); + + store + .mark_vector_index_backfilled("t", "Item", generation, "v2|embed-a") + .await + .unwrap(); + store + .delete_spec("t", "Item") + .await + .expect("delete must tombstone authority even without a specs row"); + assert!( + store + .begin_vector_index_reconciliation("t", "Item", "v2|embed-a", 100, &fingerprint_a) + .await + .is_err(), + "a compatibility authority tombstone must reject the formerly authoritative writer" + ); + assert!( + store + .begin_vector_index_reconciliation("t", "Item", "v2|", 1, "absent:v1") + .await + .unwrap() + > generation + ); +} + +#[tokio::test] +async fn stale_vector_writer_cannot_advance_journal_or_replace_reconciled_rows() { + let store = make_store("vector-stale-writer-fingerprint").await; + let csdl = ""; + let ioa_a = "[automaton]\nname = \"Item\"\n# declaration-a\n"; + let ioa_b = "[automaton]\nname = \"Item\"\n# declaration-b\n"; + let fingerprint_a = crate::spec_content_hash(ioa_a); + let fingerprint_b = crate::spec_content_hash(ioa_b); + let row_a = EntityVectorRow { + decl_name: "embed-a".to_string(), + model_tag: "model-a".to_string(), + vector: vec![1.0, 0.0], + }; + let row_b = EntityVectorRow { + decl_name: "embed-b".to_string(), + model_tag: "model-b".to_string(), + vector: vec![0.0, 1.0], + }; + let persistence_id = "t:Item:item-stale-writer"; + + store + .upsert_spec("t", "Item", ioa_a, csdl, &fingerprint_a) + .await + .unwrap(); + store.commit_specs("t").await.unwrap(); + let generation_a = store + .begin_vector_index_reconciliation("t", "Item", "v2|embed-a", 1, &fingerprint_a) + .await + .unwrap(); + let missing_fingerprint_id = "t:Item:item-missing-fingerprint"; + let missing_fingerprint_error = store + .append_with_index_rows( + missing_fingerprint_id, + 0, + &[test_envelope("Created", serde_json::json!({}))], + &[], + std::slice::from_ref(&row_a), + true, + None, + ) + .await + .unwrap_err(); + assert!( + matches!( + &missing_fingerprint_error, + PersistenceError::Storage(message) + if message.contains("requires a spec declaration fingerprint") + ), + "unexpected missing-fingerprint error: {missing_fingerprint_error:?}" + ); + assert!( + store + .read_events(missing_fingerprint_id, 0) + .await + .unwrap() + .is_empty() + ); + store + .append_with_index_rows( + persistence_id, + 0, + &[test_envelope("Created", serde_json::json!({}))], + &[], + std::slice::from_ref(&row_a), + true, + Some(&fingerprint_a), + ) + .await + .unwrap(); + + store + .upsert_spec("t", "Item", ioa_b, csdl, &fingerprint_b) + .await + .unwrap(); + store.commit_specs("t").await.unwrap(); + let generation_b = store + .begin_vector_index_reconciliation("t", "Item", "v2|embed-b", 2, &fingerprint_b) + .await + .unwrap(); + assert!(generation_b > generation_a); + store + .backfill_entity_vectors( + "t", + "Item", + "item-stale-writer", + generation_b, + 1, + std::slice::from_ref(&row_b), + ) + .await + .unwrap(); + store + .mark_vector_index_backfilled("t", "Item", generation_b, "v2|embed-b") + .await + .unwrap(); + + let fingerprinted_non_vector_id = "t:Item:item-fingerprinted-non-vector"; + let fingerprinted_non_vector_error = store + .append_with_index_rows( + fingerprinted_non_vector_id, + 0, + &[test_envelope("Created", serde_json::json!({}))], + &[], + &[], + false, + Some(&fingerprint_a), + ) + .await + .unwrap_err(); + assert!( + matches!( + &fingerprinted_non_vector_error, + PersistenceError::Storage(message) + if message.contains("stale vector declaration fingerprint") + ), + "unexpected fingerprinted non-vector error: {fingerprinted_non_vector_error:?}" + ); + assert!( + store + .read_events(fingerprinted_non_vector_id, 0) + .await + .unwrap() + .is_empty(), + "a fingerprinted append must not bypass transactional validation" + ); + + let stale_error = store + .append_with_index_rows( + persistence_id, + 1, + &[test_envelope("StaleUpdated", serde_json::json!({}))], + &[], + std::slice::from_ref(&row_a), + true, + Some(&fingerprint_a), + ) + .await + .unwrap_err(); + assert!( + matches!( + &stale_error, + PersistenceError::Storage(message) + if message.contains("stale vector declaration fingerprint") + ), + "unexpected stale-writer error: {stale_error:?}" + ); + + let batch_error = store + .append_batch(&[ + PersistenceAppend { + persistence_id: "t:Audit:audit-stale-writer".to_string(), + expected_sequence: 0, + events: vec![test_envelope("Recorded", serde_json::json!({}))], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, + }, + PersistenceAppend { + persistence_id: persistence_id.to_string(), + expected_sequence: 1, + events: vec![test_envelope("StaleBatchUpdated", serde_json::json!({}))], + vector_rows: vec![row_a], + reconcile_vectors: true, + spec_declaration_fingerprint: Some(fingerprint_a), + }, + ]) + .await + .unwrap_err(); + assert!( + matches!( + &batch_error, + PersistenceError::Storage(message) + if message.contains("stale vector declaration fingerprint") + ), + "unexpected stale batch-writer error: {batch_error:?}" + ); + + assert_eq!(store.read_events(persistence_id, 0).await.unwrap().len(), 1); + assert!( + store + .read_events("t:Audit:audit-stale-writer", 0) + .await + .unwrap() + .is_empty(), + "batch preflight must reject the stale writer before any journal changes" + ); + assert!( + store + .vector_candidates("t", "Item", "embed-a", "model-a", 10) + .await + .unwrap() + .is_empty(), + "the stale declaration must not reinstall its vector row" + ); + assert_eq!( + store + .vector_candidates("t", "Item", "embed-b", "model-b", 10) + .await + .unwrap()[0] + .vector, + row_b.vector + ); + assert_eq!( + store.vector_index_backfilled_types("t").await.unwrap(), + vec![("Item".to_string(), "v2|embed-b".to_string())] + ); +} + +#[tokio::test] +async fn deleted_spec_authority_survives_reopen_and_orders_readd() { + let url = sqlite_test_url("vector-deletion-authority-reopen"); + let ioa_source = "[automaton]\nname = \"Item\"\n# deletion-authority\n"; + let fingerprint = crate::spec_content_hash(ioa_source); + let store = TursoEventStore::new(&url, None).await.unwrap(); + store + .upsert_spec( + "t", + "Item", + ioa_source, + "", + &fingerprint, + ) + .await + .unwrap(); + store.commit_specs("t").await.unwrap(); + let present_generation = store + .begin_vector_index_reconciliation("t", "Item", "v2|embed", 1, &fingerprint) + .await + .unwrap(); + store + .mark_vector_index_backfilled("t", "Item", present_generation, "v2|embed") + .await + .unwrap(); + + store.delete_spec("t", "Item").await.unwrap(); + assert!( + store + .vector_index_backfilled_types("t") + .await + .unwrap() + .is_empty(), + "spec deletion must atomically withdraw the old completion claim" + ); + assert!( + store + .begin_vector_index_reconciliation("t", "Item", "v2|embed", 99, &fingerprint) + .await + .is_err(), + "the deleted spec fingerprint must lose authority immediately" + ); + let absent_generation = store + .begin_vector_index_reconciliation("t", "Item", "v2|", 1, "absent:v1") + .await + .unwrap(); + assert!(absent_generation > present_generation); + drop(store); + + let reopened = TursoEventStore::new(&url, None).await.unwrap(); + let resumed_generation = reopened + .begin_vector_index_reconciliation("t", "Item", "v2|", 1, "absent:v1") + .await + .unwrap(); + assert_eq!( + resumed_generation, absent_generation, + "restart must resume the deletion generation instead of allocating by process-local order" + ); + reopened + .mark_vector_index_backfilled("t", "Item", resumed_generation, "v2|") + .await + .unwrap(); + + reopened + .upsert_spec( + "t", + "Item", + ioa_source, + "", + &fingerprint, + ) + .await + .unwrap(); + reopened.commit_specs("t").await.unwrap(); + let readded_generation = reopened + .begin_vector_index_reconciliation("t", "Item", "v2|embed", 1, &fingerprint) + .await + .unwrap(); + assert!( + readded_generation > resumed_generation, + "hard delete followed by identical re-add must retain monotonic authority" + ); + assert!( + reopened + .mark_vector_index_backfilled("t", "Item", resumed_generation, "v2|") + .await + .is_err(), + "the completed absence generation must be fenced by the re-add" + ); +} + +#[tokio::test] +async fn concurrent_reopen_cannot_miss_declaration_authority_updates() { + let url = sqlite_test_url("concurrent-trigger-reinstall"); + let store = TursoEventStore::new(&url, None) + .await + .expect("create initial store"); + + for revision in 0..25 { + let ioa_source = format!("[automaton]\nname = \"Item\"\n# revision {revision}\n"); + let fingerprint = crate::spec_content_hash(&ioa_source); + let reopen = TursoEventStore::new(&url, None); + let publish = async { + store + .upsert_spec( + "tenant", + "Item", + &ioa_source, + "", + &fingerprint, + ) + .await?; + store.commit_specs("tenant").await + }; + let (reopened, published) = tokio::join!(reopen, publish); + reopened.expect("concurrent reopen must finish"); + published.expect("concurrent spec publication must finish"); + + let conn = store.configured_connection().await.unwrap(); + let mut rows = conn + .query( + "SELECT declaration_fingerprint FROM spec_declaration_authority \ + WHERE tenant = 'tenant' AND entity_type = 'Item'", + (), + ) + .await + .unwrap(); + let row = rows + .next() + .await + .unwrap() + .expect("declaration authority row"); + assert_eq!(row.get::(0).unwrap(), fingerprint); + } +} diff --git a/crates/temper-store-turso/src/store/tests/mod.rs b/crates/temper-store-turso/src/store/tests/mod.rs index f75aa4c01..bfe791c2f 100644 --- a/crates/temper-store-turso/src/store/tests/mod.rs +++ b/crates/temper-store-turso/src/store/tests/mod.rs @@ -9,6 +9,9 @@ use temper_runtime::persistence::{ use super::{PublishedArtifactUpsert, QueryProjectionUpsert, TursoEventStore}; use crate::TursoSpecVerificationUpdate; +mod declaration_authority; +mod spec_catalog; + fn test_envelope(event_type: &str, payload: serde_json::Value) -> PersistenceEnvelope { PersistenceEnvelope { sequence_nr: 0, @@ -39,6 +42,23 @@ async fn make_store(test_name: &str) -> TursoEventStore { .expect("create store") } +async fn install_vector_spec(store: &TursoEventStore, revision_label: &str) -> String { + let ioa_source = format!("[automaton]\nname = \"Item\"\n# {revision_label}\n"); + let fingerprint = crate::spec_content_hash(&ioa_source); + store + .upsert_spec( + "t", + "Item", + &ioa_source, + "", + &fingerprint, + ) + .await + .expect("persist vector spec"); + store.commit_specs("t").await.expect("commit vector spec"); + fingerprint +} + #[tokio::test] async fn append_and_read_events_roundtrip() { let store = make_store("append-read").await; @@ -67,16 +87,21 @@ async fn append_and_read_events_roundtrip() { } #[tokio::test] -async fn vector_index_write_behind_candidates_and_partitioning() { - // ADR-0155: Turso maintains entity_vector_index write-behind (event first, index - // follows). A candidate scan returns the partition's vectors in entity_id order, - // partitioned by model tag; a raw kNN read never sees another model's vectors. +async fn vector_index_co_commit_candidates_and_partitioning() { + // ADR-0181: Turso co-commits entity_vector_index with the event journal. A + // candidate scan returns vectors in entity_id order, partitioned by model tag; + // a raw kNN read never sees another model's vectors. let store = make_store("vector-index").await; + let fingerprint = install_vector_spec(&store, "vector-index-v1").await; let row = |decl: &str, model: &str, v: Vec| EntityVectorRow { decl_name: decl.to_string(), model_tag: model.to_string(), vector: v, }; + let generation = store + .begin_vector_index_reconciliation("t", "Item", "embed", 1, &fingerprint) + .await + .unwrap(); store .append_with_index_rows( @@ -86,6 +111,7 @@ async fn vector_index_write_behind_candidates_and_partitioning() { &[], &[row("embed", "m1", vec![0.0, 1.0])], true, + Some(&fingerprint), ) .await .unwrap(); @@ -97,6 +123,7 @@ async fn vector_index_write_behind_candidates_and_partitioning() { &[], &[row("embed", "m1", vec![1.0, 0.0])], true, + Some(&fingerprint), ) .await .unwrap(); @@ -109,6 +136,7 @@ async fn vector_index_write_behind_candidates_and_partitioning() { &[], &[row("embed", "m2", vec![1.0, 0.0])], true, + Some(&fingerprint), ) .await .unwrap(); @@ -126,7 +154,14 @@ async fn vector_index_write_behind_candidates_and_partitioning() { // Upsert: re-writing item-a's vector replaces (no duplicate row). store - .backfill_entity_vectors("t", "Item", "item-a", &[row("embed", "m1", vec![0.5, 0.5])]) + .backfill_entity_vectors( + "t", + "Item", + "item-a", + generation, + 1, + &[row("embed", "m1", vec![0.5, 0.5])], + ) .await .unwrap(); let candidates = store @@ -138,7 +173,7 @@ async fn vector_index_write_behind_candidates_and_partitioning() { // Watermark roundtrip + resumable id listing. store - .mark_vector_index_backfilled("t", "Item", "embed") + .mark_vector_index_backfilled("t", "Item", generation, "embed") .await .unwrap(); assert_eq!( @@ -158,13 +193,18 @@ async fn vector_index_reconcile_purges_on_delete_and_empty_rows() { // ADR-0155: a delete/clear reconciles to an empty row set, purging the entity's // vector rows (the turso-side "remove" cleanup) so it is never ranked again. let store = make_store("vector-purge").await; + let fingerprint = install_vector_spec(&store, "vector-purge-v1").await; let row = |v: Vec| EntityVectorRow { decl_name: "embed".to_string(), model_tag: "m1".to_string(), vector: v, }; + let generation = store + .begin_vector_index_reconciliation("t", "Item", "embed", 1, &fingerprint) + .await + .unwrap(); - // Write-behind reconcile with a row, then a delete transition (empty rows). + // Co-commit a row, then a delete transition (empty rows). store .append_with_index_rows( "t:Item:item-a", @@ -173,6 +213,7 @@ async fn vector_index_reconcile_purges_on_delete_and_empty_rows() { &[], std::slice::from_ref(&row(vec![1.0, 0.0])), true, + Some(&fingerprint), ) .await .unwrap(); @@ -193,6 +234,7 @@ async fn vector_index_reconcile_purges_on_delete_and_empty_rows() { &[], &[], true, + Some(&fingerprint), ) .await .unwrap(); @@ -207,7 +249,7 @@ async fn vector_index_reconcile_purges_on_delete_and_empty_rows() { // The explicit backfill purge (empty rows) is idempotent. store - .backfill_entity_vectors("t", "Item", "item-a", &[]) + .backfill_entity_vectors("t", "Item", "item-a", generation, 2, &[]) .await .unwrap(); assert!( @@ -219,6 +261,515 @@ async fn vector_index_reconcile_purges_on_delete_and_empty_rows() { ); } +#[tokio::test] +async fn vector_index_failure_never_commits_journal_without_index() { + let store = make_store("vector-atomicity").await; + let fingerprint = install_vector_spec(&store, "vector-atomicity-v1").await; + let conn = store.configured_connection().await.unwrap(); + conn.execute( + "CREATE TRIGGER reject_vector_insert \ + BEFORE INSERT ON entity_vector_index \ + BEGIN SELECT RAISE(ABORT, 'forced vector-index write failure'); END", + (), + ) + .await + .unwrap(); + + let persistence_id = "t:Item:item-atomic"; + let result = store + .append_with_index_rows( + persistence_id, + 0, + &[test_envelope("Create", serde_json::json!({}))], + &[], + &[EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }], + true, + Some(&fingerprint), + ) + .await; + let journal = store.read_events(persistence_id, 0).await.unwrap(); + + assert!( + result.is_err() && journal.is_empty(), + "a rejected vector write must roll back its journal append; result={result:?}, journal_len={}", + journal.len() + ); +} + +#[tokio::test] +async fn pre_reconciliation_live_vector_type_remains_discoverable() { + let store = make_store("vector-pre-generation-discovery").await; + let fingerprint = install_vector_spec(&store, "vector-pre-generation-v1").await; + store + .append_with_index_rows( + "t:Item:item-before-generation", + 0, + &[test_envelope("Created", serde_json::json!({}))], + &[], + &[EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }], + true, + Some(&fingerprint), + ) + .await + .unwrap(); + + assert_eq!( + store.vector_reconciliation_entity_types("t").await.unwrap(), + vec!["Item".to_string()], + "generation-zero fences must keep remove-all reconciliation discoverable" + ); +} + +#[tokio::test] +async fn composite_vector_index_failure_rolls_back_every_journal() { + let store = make_store("vector-composite-atomicity").await; + let fingerprint = install_vector_spec(&store, "vector-composite-atomicity-v1").await; + let conn = store.configured_connection().await.unwrap(); + conn.execute( + "CREATE TRIGGER reject_composite_vector_insert \ + BEFORE INSERT ON entity_vector_index \ + BEGIN SELECT RAISE(ABORT, 'forced composite vector-index write failure'); END", + (), + ) + .await + .unwrap(); + + let item_persistence_id = "t:Item:item-composite-atomic"; + let audit_persistence_id = "t:Audit:audit-composite-atomic"; + let result = store + .append_batch(&[ + PersistenceAppend { + persistence_id: item_persistence_id.to_string(), + expected_sequence: 0, + events: vec![test_envelope("Created", serde_json::json!({}))], + vector_rows: vec![EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }], + reconcile_vectors: true, + spec_declaration_fingerprint: Some(fingerprint.clone()), + }, + PersistenceAppend { + persistence_id: audit_persistence_id.to_string(), + expected_sequence: 0, + events: vec![test_envelope("Recorded", serde_json::json!({}))], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, + }, + ]) + .await; + + assert!(result.is_err(), "the vector failure must reject the batch"); + assert!( + store + .read_events(item_persistence_id, 0) + .await + .unwrap() + .is_empty(), + "the vector-owning journal must roll back" + ); + assert!( + store + .read_events(audit_persistence_id, 0) + .await + .unwrap() + .is_empty(), + "every other journal in the composite batch must roll back" + ); +} + +#[tokio::test] +async fn stale_vector_backfill_cannot_overwrite_or_resurrect_turso_write() { + let store = make_store("vector-monotonic").await; + let fingerprint = install_vector_spec(&store, "vector-monotonic-v1").await; + let row = |v: Vec| EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: v, + }; + let persistence_id = "t:Item:item-race"; + let generation = store + .begin_vector_index_reconciliation("t", "Item", "embed", 1, &fingerprint) + .await + .unwrap(); + + store + .append_with_index_rows( + persistence_id, + 0, + &[test_envelope("Created", serde_json::json!({}))], + &[], + &[row(vec![1.0, 0.0])], + true, + Some(&fingerprint), + ) + .await + .unwrap(); + store + .append_with_index_rows( + persistence_id, + 1, + &[test_envelope("Updated", serde_json::json!({}))], + &[], + &[row(vec![0.0, 1.0])], + true, + Some(&fingerprint), + ) + .await + .unwrap(); + store + .backfill_entity_vectors( + "t", + "Item", + "item-race", + generation, + 1, + &[row(vec![1.0, 0.0])], + ) + .await + .unwrap(); + assert_eq!( + store + .vector_candidates("t", "Item", "embed", "m1", 10) + .await + .unwrap()[0] + .vector, + vec![0.0, 1.0], + "the sequence-1 rebuild must not replace the sequence-2 live row" + ); + + store + .append_with_index_rows( + persistence_id, + 2, + &[test_envelope("Deleted", serde_json::json!({}))], + &[], + &[], + true, + Some(&fingerprint), + ) + .await + .unwrap(); + store + .backfill_entity_vectors( + "t", + "Item", + "item-race", + generation, + 2, + &[row(vec![0.0, 1.0])], + ) + .await + .unwrap(); + assert!( + store + .vector_candidates("t", "Item", "embed", "m1", 10) + .await + .unwrap() + .is_empty(), + "the retained sequence-3 fence must prevent stale resurrection" + ); + + store + .backfill_entity_vectors("t", "Item", "item-race", generation, 3, &[]) + .await + .unwrap(); + store + .backfill_entity_vectors("t", "Item", "item-race", generation, 3, &[]) + .await + .unwrap(); + assert!( + !store + .list_entity_ids_by_type("t", "Item") + .await + .unwrap() + .iter() + .any(|entity_id| entity_id == "item-race"), + "active listing excludes the deleted stream" + ); + assert!( + store + .list_vector_repair_entity_ids("t", "Item") + .await + .unwrap() + .iter() + .any(|entity_id| entity_id == "item-race"), + "repair enumeration must retain deleted journal streams" + ); +} + +#[tokio::test] +async fn composite_batch_co_commits_vector_fence_before_delayed_repair() { + let store = make_store("vector-composite-batch").await; + let fingerprint = install_vector_spec(&store, "vector-composite-v1").await; + let generation = store + .begin_vector_index_reconciliation("t", "Item", "embed", 1, &fingerprint) + .await + .unwrap(); + let stale_row = EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }; + let live_row = EntityVectorRow { + decl_name: "embed".to_string(), + model_tag: "m1".to_string(), + vector: vec![0.0, 1.0], + }; + store + .append_with_index_rows( + "t:Item:item-batch", + 0, + &[test_envelope("Created", serde_json::json!({}))], + &[], + std::slice::from_ref(&stale_row), + true, + Some(&fingerprint), + ) + .await + .unwrap(); + + store + .append_batch(&[ + PersistenceAppend { + persistence_id: "t:Item:item-batch".to_string(), + expected_sequence: 1, + events: vec![test_envelope("CompositeUpdated", serde_json::json!({}))], + vector_rows: vec![live_row.clone()], + reconcile_vectors: true, + spec_declaration_fingerprint: Some(fingerprint.clone()), + }, + PersistenceAppend { + persistence_id: "t:Audit:audit-batch".to_string(), + expected_sequence: 0, + events: vec![test_envelope("Recorded", serde_json::json!({}))], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, + }, + ]) + .await + .unwrap(); + store + .backfill_entity_vectors("t", "Item", "item-batch", generation, 1, &[stale_row]) + .await + .unwrap(); + + assert_eq!( + store + .vector_candidates("t", "Item", "embed", "m1", 10) + .await + .unwrap()[0] + .vector, + live_row.vector + ); + assert_eq!( + store + .read_events("t:Audit:audit-batch", 0) + .await + .unwrap() + .len(), + 1 + ); +} + +#[tokio::test] +async fn newer_reconciliation_generation_rejects_older_rows_and_watermark() { + let store = make_store("vector-generation-order").await; + let fingerprint_old = install_vector_spec(&store, "vector-generation-old").await; + let old_generation = store + .begin_vector_index_reconciliation("t", "Item", "old", 1, &fingerprint_old) + .await + .unwrap(); + store + .append_with_index_rows( + "t:Item:item-generation", + 0, + &[test_envelope("Created", serde_json::json!({}))], + &[], + &[EntityVectorRow { + decl_name: "old".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }], + true, + Some(&fingerprint_old), + ) + .await + .unwrap(); + + let fingerprint_new = install_vector_spec(&store, "vector-generation-new").await; + let new_generation = store + .begin_vector_index_reconciliation("t", "Item", "new", 2, &fingerprint_new) + .await + .unwrap(); + store + .backfill_entity_vectors( + "t", + "Item", + "item-generation", + new_generation, + 1, + &[EntityVectorRow { + decl_name: "new".to_string(), + model_tag: "m2".to_string(), + vector: vec![0.0, 1.0], + }], + ) + .await + .unwrap(); + store + .mark_vector_index_backfilled("t", "Item", new_generation, "new") + .await + .unwrap(); + + assert!( + store + .backfill_entity_vectors( + "t", + "Item", + "item-generation", + old_generation, + 1, + &[EntityVectorRow { + decl_name: "old".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }], + ) + .await + .is_err() + ); + assert!( + store + .mark_vector_index_backfilled("t", "Item", old_generation, "old") + .await + .is_err() + ); + assert_eq!( + store.vector_index_backfilled_types("t").await.unwrap(), + vec![("Item".to_string(), "new".to_string())] + ); + assert!( + store + .vector_candidates("t", "Item", "old", "m1", 10) + .await + .unwrap() + .is_empty() + ); +} + +#[tokio::test] +async fn beginning_reconciliation_withdraws_the_previous_completion_claim() { + let store = make_store("vector-generation-watermark-invalidation").await; + let fingerprint_a = install_vector_spec(&store, "watermark-a-1").await; + let row_a = EntityVectorRow { + decl_name: "embed-a".to_string(), + model_tag: "m1".to_string(), + vector: vec![1.0, 0.0], + }; + let row_b = EntityVectorRow { + decl_name: "embed-b".to_string(), + model_tag: "m2".to_string(), + vector: vec![0.0, 1.0], + }; + let first_a = store + .begin_vector_index_reconciliation("t", "Item", "v2|a", 1, &fingerprint_a) + .await + .unwrap(); + store + .append_with_index_rows( + "t:Item:item-signature-race", + 0, + &[test_envelope("Created", serde_json::json!({}))], + &[], + std::slice::from_ref(&row_a), + true, + Some(&fingerprint_a), + ) + .await + .unwrap(); + store + .mark_vector_index_backfilled("t", "Item", first_a, "v2|a") + .await + .unwrap(); + + let fingerprint_b = install_vector_spec(&store, "watermark-b").await; + let generation_b = store + .begin_vector_index_reconciliation("t", "Item", "v2|b", 2, &fingerprint_b) + .await + .unwrap(); + assert!( + store + .vector_index_backfilled_types("t") + .await + .unwrap() + .is_empty(), + "beginning B must atomically withdraw A's completion watermark" + ); + assert_eq!( + store.vector_reconciliation_entity_types("t").await.unwrap(), + vec!["Item".to_string()], + "the in-progress type must remain discoverable without its watermark" + ); + + let fingerprint_a = install_vector_spec(&store, "watermark-a-2").await; + let second_a = store + .begin_vector_index_reconciliation("t", "Item", "v2|a", 3, &fingerprint_a) + .await + .unwrap(); + assert!(second_a > generation_b); + store + .backfill_entity_vectors( + "t", + "Item", + "item-signature-race", + second_a, + 1, + std::slice::from_ref(&row_a), + ) + .await + .unwrap(); + store + .mark_vector_index_backfilled("t", "Item", second_a, "v2|a") + .await + .unwrap(); + + assert!( + store + .backfill_entity_vectors( + "t", + "Item", + "item-signature-race", + generation_b, + 1, + &[row_b], + ) + .await + .is_err() + ); + assert!( + store + .mark_vector_index_backfilled("t", "Item", generation_b, "v2|b") + .await + .is_err() + ); + assert_eq!( + store.vector_index_backfilled_types("t").await.unwrap(), + vec![("Item".to_string(), "v2|a".to_string())] + ); +} + #[tokio::test] async fn append_with_wrong_sequence_fails_with_concurrency_violation() { let store = make_store("concurrency").await; @@ -282,6 +833,9 @@ async fn append_batch_zero_sequence_detects_existing_stream_by_unique_key() { "OrderUpdated", serde_json::json!({ "step": 2 }), )], + vector_rows: Vec::new(), + reconcile_vectors: false, + spec_declaration_fingerprint: None, }]) .await .unwrap_err(); @@ -1912,10 +2466,21 @@ async fn load_verification_cache_ignores_uncommitted_specs() { .upsert_spec(&tenant, "Issue", ioa_source, csdl_xml, content_hash) .await .expect("upsert uncommitted spec"); + let cache = store + .load_verification_cache(&tenant) + .await + .expect("load verification cache"); + assert!( + !cache.contains_key("Issue"), + "uncommitted specs must not be used to skip bootstrap persistence" + ); + store - .persist_spec_verification( + .commit_verified_spec( &tenant, "Issue", + content_hash, + csdl_xml, TursoSpecVerificationUpdate { status: "passed", verified: true, @@ -1925,18 +2490,7 @@ async fn load_verification_cache_ignores_uncommitted_specs() { }, ) .await - .expect("persist verification"); - - let cache = store - .load_verification_cache(&tenant) - .await - .expect("load verification cache"); - assert!( - !cache.contains_key("Issue"), - "uncommitted specs must not be used to skip bootstrap persistence" - ); - - store.commit_specs(&tenant).await.expect("commit spec"); + .expect("verify and commit spec"); let cache = store .load_verification_cache(&tenant) .await diff --git a/crates/temper-store-turso/src/store/tests/spec_catalog.rs b/crates/temper-store-turso/src/store/tests/spec_catalog.rs new file mode 100644 index 000000000..6104fb3bd --- /dev/null +++ b/crates/temper-store-turso/src/store/tests/spec_catalog.rs @@ -0,0 +1,150 @@ +use super::*; + +#[tokio::test] +async fn late_verifier_cannot_publish_newer_staged_bytes() { + let url = sqlite_test_url("late-verifier-exact-catalog"); + let store = TursoEventStore::new(&url, None).await.expect("open store"); + let csdl = ""; + let source_a = "[automaton]\nname = \"Item\"\n# candidate-a\n"; + let source_b = "[automaton]\nname = \"Item\"\n# candidate-b\n"; + let fingerprint_a = crate::spec_content_hash(source_a); + let fingerprint_b = crate::spec_content_hash(source_b); + store + .upsert_spec("t", "Item", source_a, csdl, &fingerprint_a) + .await + .expect("stage candidate A"); + store + .upsert_spec("t", "Item", source_b, csdl, &fingerprint_b) + .await + .expect("newer candidate B replaces staging"); + + let error = store + .persist_verified_spec_catalog_update( + "t", + &[("Item", fingerprint_a.as_str(), csdl)], + &[], + true, + None, + ) + .await + .expect_err("candidate A verification must not publish candidate B"); + assert!(error.to_string().contains("fingerprint changed")); + assert!( + store + .load_specs() + .await + .expect("load committed specs") + .is_empty(), + "no committed catalog may be created from mismatched staged bytes" + ); +} + +#[tokio::test] +async fn concurrent_replica_replacements_commit_one_complete_catalog() { + let url = sqlite_test_url("concurrent-spec-catalog-replacement"); + let store_a = TursoEventStore::new(&url, None) + .await + .expect("open first replica store"); + let store_b = TursoEventStore::new(&url, None) + .await + .expect("open second replica store"); + let csdl = ""; + let source_a = "[automaton]\nname = \"ItemA\"\n"; + let source_b = "[automaton]\nname = \"ItemB\"\n"; + let fingerprint_a = crate::spec_content_hash(source_a); + let fingerprint_b = crate::spec_content_hash(source_b); + let specs_a = [("ItemA", source_a, fingerprint_a.as_str())]; + let specs_b = [("ItemB", source_b, fingerprint_b.as_str())]; + + let (result_a, result_b) = tokio::join!( + store_a.persist_spec_catalog_update("t", &specs_a, csdl, &[], true, None), + store_b.persist_spec_catalog_update("t", &specs_b, csdl, &[], true, None), + ); + result_a.expect("first replica replacement must commit"); + result_b.expect("second replica replacement must commit"); + + drop(store_a); + drop(store_b); + let reopened = TursoEventStore::new(&url, None) + .await + .expect("reopen catalog after both replacements"); + let committed = reopened + .load_specs() + .await + .expect("load committed catalog") + .into_iter() + .filter(|row| row.tenant == "t") + .map(|row| row.entity_type) + .collect::>(); + assert!( + committed == ["ItemA"] || committed == ["ItemB"], + "the final durable catalog must be one serialized replacement, got {committed:?}" + ); + assert_eq!( + reopened + .spec_replacement_entity_types("t") + .await + .expect("load present authority"), + committed, + "reopen must recover the same single authoritative catalog" + ); +} + +#[tokio::test] +async fn merge_without_constraints_preserves_them_across_restart_and_replace_clears_them() { + let url = sqlite_test_url("merge-preserves-spec-catalog-constraints"); + let store = TursoEventStore::new(&url, None).await.expect("open store"); + let csdl = ""; + let source_a = "[automaton]\nname = \"ItemA\"\n"; + let source_b = "[automaton]\nname = \"ItemB\"\n"; + let fingerprint_a = crate::spec_content_hash(source_a); + let fingerprint_b = crate::spec_content_hash(source_b); + let specs_a = [("ItemA", source_a, fingerprint_a.as_str())]; + let specs_b = [("ItemB", source_b, fingerprint_b.as_str())]; + let constraints = r#"version = 1 +default_delete_policy = "restrict" + +[[invariant]] +name = "payment_must_be_captured" +kind = "hard" +on = "Order.Submit" +assert = 'related(Payment, payment_id).status in ["Captured"]' +"#; + + store + .persist_spec_catalog_update("t", &specs_a, csdl, &[], true, Some(constraints)) + .await + .expect("seed replacement with constraints"); + store + .persist_spec_catalog_update("t", &specs_b, csdl, &[], false, None) + .await + .expect("merge without constraints"); + drop(store); + + let reopened = TursoEventStore::new(&url, None) + .await + .expect("reopen after merge"); + let persisted = reopened + .load_tenant_constraints() + .await + .expect("load constraints after restart"); + assert_eq!(persisted.len(), 1); + assert_eq!(persisted[0].tenant, "t"); + assert_eq!(persisted[0].cross_invariants_toml, constraints); + + reopened + .persist_spec_catalog_update("t", &specs_a, csdl, &[], true, None) + .await + .expect("constraint-free replacement"); + drop(reopened); + let final_store = TursoEventStore::new(&url, None) + .await + .expect("reopen after replacement"); + assert!( + final_store + .load_tenant_constraints() + .await + .expect("load cleared constraints") + .is_empty() + ); +} diff --git a/docs/adrs/0181-monotonic-vector-reconciliation.md b/docs/adrs/0181-monotonic-vector-reconciliation.md new file mode 100644 index 000000000..4106c1ec2 --- /dev/null +++ b/docs/adrs/0181-monotonic-vector-reconciliation.md @@ -0,0 +1,432 @@ +# ADR-0181: Monotonic vector reconciliation + +- Status: Proposed +- Date: 2026-07-14 +- Deciders: Temper core maintainers +- Related: + - ADR-0155: Declared vector access path + - ADR-0153: Declared composite-key index + - ARN-216: Vector backfill races live writes and can permanently mark a stale index complete + - ARN-201: Canonical append/projection transaction contract + - `crates/temper-runtime/src/persistence/mod.rs` + - `crates/temper-server/src/state/projection_backfill/vector_index.rs` + - `crates/temper-store-{sim,postgres,turso}` + +## Context + +ADR-0155 made `entity_vector_index` derived state and gave each row a journal +`sequence_nr`, but the backfill contract does not carry the sequence of the state it +read. Postgres and Turso backfill therefore delete every row for an entity and insert +the replacement at sequence zero. The simulation store does not retain vector-index +versions at all. + +That loses the ordering proof between a background rebuild and a live append. A +backfill can read state at sequence N, a live append can co-commit vectors for N+1, and +the delayed backfill can then replace N+1 with N. The backfill records its type +watermark after the stale replacement, so later starts skip the type and the stale +ranking becomes durable. + +Row sequence numbers alone cannot close the race. A delete or cleared vector must +remove every candidate row. If the removed rows were the only place that retained +sequence N+1, delayed work from N could reinsert a vector after the purge. Correct +whole-entity replacement therefore needs an ordering record that survives an empty +row set. + +Two existing optimizations compound the problem: + +- A first-time backfill skips any entity that already has one vector row, even when + that row represents an older journal sequence. +- An active entity with no currently usable vector is classified as skippable instead + of reconciling an empty row set, so a failed earlier purge is not repaired. + +Turso also acknowledges the journal append before its separate vector write-behind. +After retry exhaustion it logs the vector failure but returns append success. An +already-current watermark then prevents startup backfill from repairing that entity. + +Two more journal-writing paths need the same ordering contract. Composite actions +append several streams atomically through `append_batch`; carrying only journal events +there would let those streams advance without advancing their vector fences. Spec +reconciliation can also overlap: sequence ordering alone cannot distinguish two +different declaration sets rebuilt from the same journal sequence, so an older rebuild +could replace a newer declaration set and then publish its stale watermark. + +## Decision + +### Sub-Decision 1: Fence whole-entity replacement with a durable sequence row + +Every indexing backend will maintain one +`entity_vector_index_version (tenant, entity_type, entity_id, +reconciliation_generation, sequence_nr)` row per entity whose vector state has been +reconciled. The row is retained when the entity has no candidate vectors, including +deletion and cleared-vector purges. + +`EventStore::backfill_entity_vectors` will accept the journal sequence observed by the +caller. In one backend transaction it will: + +1. reject a stale reconciliation generation; +2. advance the entity's version row when its generation is newer, or when its + generation matches and `observed_sequence >= sequence_nr`; +3. return success without changing candidates when a newer sequence is already stored; +4. delete all candidate rows for the entity; +5. insert the observed rows with `sequence_nr = observed_sequence`; and +6. commit the version fence and candidates together. + +Equal-sequence replacement is allowed so replay is idempotent and can repair a +partially initialized derived index. A lower-sequence replacement is also a successful +outcome: the durable fence proves that the index already represents newer state. + +**Why this approach**: vector declarations are reconciled as one post-transition +entity state, not as independent fields. One retained entity-level fence protects +model-tag changes, declaration removals, and empty purges without inventing sentinel +candidate rows. + +### Sub-Decision 2: Order declaration-set reconciliation with a durable generation + +Every authoritative indexing backend will maintain one +`entity_vector_reconciliation_generation (tenant, entity_type, generation, +declaration_revision, declaration_fingerprint, vector_set)` row. A caller supplies its +process-local monotonic tenant revision plus the fingerprint of the IOA source from +which it snapshotted declarations. + +Postgres and Turso additionally maintain +`spec_declaration_authority (tenant, entity_type, revision, ioa_source, +declaration_fingerprint, present)`. +Database triggers advance this row in the same transaction as every IOA insert, +source change, and hard deletion **only when the affected catalog row is committed**. +PlatformStore writes replacement bytes to a separate `staged_specs` row. The last +committed `specs` row therefore remains restorable if verification or the process +crashes. Staging and discarded rows neither fence the still-published declaration nor +withdraw its watermark. Atomic promotion of one exact staged IOA+CSDL pair is the +publication point that advances authority. The authority row is a tombstone when +`present = false`, so its revision survives delete/re-add and process restart. A +committed spec mutation also advances an existing reconciliation generation and +withdraws its watermark immediately; stale work is fenced at the declaration commit +point, not only after the next coordinator starts. + +Persistent reconciliation uses the catalog's stored content fingerprint, falling back +to hashing authoritative IOA bytes only for migrated rows, or uses the fixed +`absent:v1` tombstone fingerprint. Validation and the journal/index mutation hold the +same authority-row barrier through commit. A truly empty compatibility store may +atomically accept its first fingerprint as authority only when neither a catalog row +nor an authority/tombstone row exists. That bootstrap never overwrites catalog truth, +and concurrent different first writers leave exactly one winner. A replica holding A +therefore cannot begin after durable B merely because its call arrives later, and an +intentional A re-add receives a strictly newer tombstone-preserved revision. The +process-local revision remains diagnostic input and is not trusted as cross-process +authority. + +Deterministic simulation mirrors the separate durable authority map. Declaration +changes use `persist_spec_declaration`; once an authority entry exists, no caller-local +revision can replace its fingerprint. Direct EventStore tests retain an empty-store +first-writer bootstrap, but append validation stages that bootstrap and publishes it +only if the complete append/batch commits. Retrying the identical declaration and +vector set reuses its generation and does not withdraw an already-valid watermark. + +Before rebuilding a mismatched declaration set, the coordinator atomically advances +that type's generation, withdraws the prior completion watermark, and receives the new +token. Withdrawing the watermark prevents a coordinator for the old signature from +observing a now-invalid completion claim and skipping. Every entity replacement and +the final watermark write carry the token and fail if it is no longer current. Live +vector writes read the current type generation and co-commit it into the entity fence +with the new journal sequence. PostgreSQL takes a shared row lock for that read: +concurrent live writers remain independent, while a generation update waits for all +earlier writers to commit. + +The in-process coordinator serializes only declaration snapshotting and durable +generation allocation. It releases that lock before journal enumeration, replay, and +row replacement, so a long rebuild does not globally serialize unrelated tenants or +types. The durable declaration revision and generation remain the cross-process and +crash boundary. A stale revision or generation is an explicit failure, not a +successful no-op, because it must prevent the stale invocation from claiming +completion. + +**Why this approach**: equal-sequence replay is necessary for idempotent repair inside +one declaration set, so sequence alone cannot order two different sets. A durable +type-level epoch makes that order explicit without coupling persistence backends to +the in-memory registry implementation. + +### Sub-Decision 3: Backfill every entity from an observed journal sequence + +State recovery will return both fields and `EntityState::sequence_nr`; deleted and +phantom outcomes will also retain the recovered sequence used for an ordered purge. +Vector repair will enumerate durable journal stream IDs, including streams whose +latest state is `Deleted`, through a repair-specific EventStore method. It will not +reuse active-entity listing, whose Postgres and Turso implementations intentionally +exclude deleted streams. + +Whenever the watermark is absent or its signature differs, the backfill will load and +reconcile every journal stream for the declared type. It will not skip an entity merely +because some candidate row already exists. Deleted streams reconcile an empty row set +at their deletion sequence, which both removes a legacy stale candidate and leaves the +version tombstone that rejects older work. + +An active entity with no valid vector/model pair will reconcile an empty row set at +its observed sequence. This repairs stale candidates left by an interrupted or older +write path. + +The watermark signature gains a reconciliation-protocol revision. Existing ADR-0155 +watermarks therefore mismatch once after rollout and force a sequence-aware rebuild +without relying on backend-specific migration state. + +The work set is the union of types with current vector declarations, types with a +stored vector-backfill watermark, and types with any durable reconciliation state +(generation rows, retained entity fences, or candidate rows). The third source is +required because beginning a generation withdraws the old watermark: if the process +crashes while reconciling an empty declaration set, the durable generation still makes +the purge discoverable on restart. It also covers generation-zero rows created by live +writes or migrated from ADR-0155 before their first formal reconciliation. A previously +covered type whose current declaration set is empty is rebuilt to an empty candidate +set across all of its journal streams and then receives the revisioned empty-set +watermark. Removing the final declaration therefore cannot leave an old watermark that +would match if the identical declaration is later re-added. + +**Why this approach**: the supported exact-scan design is explicitly bounded to about +1,000 entities per tenant. Re-reading the full type after an incomplete run is simpler +and sounder than a row-presence shortcut that cannot prove journal freshness. + +### Sub-Decision 4: Co-commit live vector state on every journal-writing path + +Postgres and the simulation store will update the version fence in the same critical +section or transaction that already commits the journal and candidate rows. + +The composite `append_batch` contract will carry each stream's complete +post-transition vector rows plus whether its declared vector set must be reconciled. +Each backend will co-commit those rows and the current reconciliation-generation fence +with every batch journal append. Empty rows are meaningful: a composite delete or +cleared vector purges candidates while retaining the fence. Backends without vector +index authority may still commit the journal batch, but cannot later advertise a +vector-reconciliation watermark. + +Turso will stop using event-first vector write-behind. Its journal, version fence, and +vector tables share the same libSQL database, so an indexed append will use the existing +immediate transaction path and commit all three together. Every spec-derived writer, +including a currently non-vector declaration, carries the fingerprint of the exact +transition-table snapshot that produced its event. The store validates that fingerprint +before any journal mutation. This prevents an old replica from advancing the journal +after a newer declaration adds, removes, or changes vectors. The actor retry path, +composite staging, native data-only create, and atomic File initial-write path all retain +their original table snapshot through commit; none re-read a hot-swapped table merely +to label old semantics with a new fingerprint. + +The single-event optimization remains available only to legacy/untyped appends that +carry no declaration fingerprint. A durable outbox is not needed while all affected +records share this transactional boundary; a future backend with a physically separate +vector store must add a pre-commit durable obligation before it can advertise +vector-index authority. + +**Why this approach**: an outbox would add a second state machine, cleanup rules, and +watermark coupling to emulate atomicity that the current Turso topology already +provides. The longer indexed-append transaction is the deliberate durability cost. + +### Sub-Decision 5: A watermark is a persisted convergence claim + +A type is reported complete only when every entity load and ordered replacement +succeeds and the generation-checked watermark write itself succeeds. A lower-sequence +replacement rejected within the current generation counts as converged because newer +durable vector state is present. A stale-generation replacement does not. + +Failure to persist the watermark logs a failure outcome; the code must not emit the +"type watermarked" completion event. The next run replays the bounded type and +converges idempotently. + +The coordinator must cross the declaration barrier before trusting an existing +watermark, then re-read completion under its short coordinator lock. This closes the +window where a spec mutation withdraws a completion claim after the coordinator's +initial tenant-wide read but before it decides to skip the type. + +### Sub-Decision 6: Full spec replacement persists omission tombstones + +For full-directory replacement, the durable spec catalog and in-memory registry form +one ordered publication. Omission discovery is part of the backend write transaction, +not a query performed before mutation. Postgres takes a tenant-scoped advisory +transaction lock; Turso begins an immediate transaction. Only after that shared lock is +held does the backend read the current catalog and present declaration authority, +upsert the incoming committed set, tombstone every omission, and update tenant +constraints. The server hot-load path and CLI startup overlay both call this exact +primitive. Concurrent replicas therefore serialize as two complete replacements; they +cannot each observe an empty catalog and commit their union. Merge-mode inline +submissions do not delete omitted types. + +The transaction returns the exact durable omissions it replaced. The server unions +those with any registry-only compatibility omissions before publishing the new +registry. Turso commits only the addressed tenant's incoming set and constraints; it +does not use a process-wide commit of unrelated staged rows. + +Startup can subsequently merge built-in agent entities into an app tenant. That phase +must publish their sources, exact fingerprints, and verification state through the +tenant's active `PlatformStore`, including shared Postgres. A Turso-only bootstrap +accessor would leave the in-memory built-ins advertised while replacement tombstones +continued to fence every Postgres writer. Each verified built-in is committed by +tenant and entity type; bootstrap must never use a tenant-wide commit that could +promote an unrelated app declaration still undergoing verification on another +Postgres replica. Verification status and commitment are finalized in one store +operation that compares both the IOA fingerprint and the exact CSDL bytes. A same-type +IOA overwrite or a same-IOA/different-CSDL overwrite by another replica therefore +fails closed instead of publishing bytes that the current bootstrap did not verify. +Multi-spec app installs promote their owned entity/hash/CSDL tuples as one atomic, +entity-name-ordered batch after the app's policy and metadata writes. A missing or +replaced staged row rolls back the whole batch, while unrelated crash-orphan staging +remains quarantined. + +A delete always leaves authority at `absent:v1`, even when compatibility first-writer +bootstrap created authority without a `specs` row. The deletion trigger/transaction +advances any existing reconciliation generation and removes its watermark. The absent +type therefore remains discoverable from durable reconciliation state after a crash, +can purge retained candidates without loading a current transition table, and cannot +be resurrected by stale writers or startup restore. + +**Why this approach**: removing a type only from the process registry is not a durable +declaration change. On restart the old catalog row would restore the type, while an +old vector watermark could suppress its purge. Ordering storage before registry +publication fails closed during hot swap and makes deletion replayable. + +### Sub-Decision 7: Registry publication preserves only live actor incarnations + +Before durable catalog mutation, the server snapshots the actor key, task UUID, and +ready supervised-incarnation epoch only for matching actors that completed +`pre_start`. The `ActorRef` exposes readiness and a monotonically increasing +`pre_start` epoch in one packed atomic value. A drop guard in the actor run future +clears readiness on normal shutdown, handler panic, and task cancellation; every +supervised restart advances the epoch even though it reuses the task UUID. After the +durable commit and while holding the actor/spec publication write lock, the server +preserves an actor only when the same key still maps to the same UUID and ready epoch. +It stops and removes actors created or restarted during the publication gap, same-key +replacement incarnations, unready or dead actors, every removed type, and every legacy +fallback actor on a tenant's first registry publication. + +Preserved actors share the registry transition-table lock and hot-swap in place. An +actor that captured the old declaration after the snapshot cannot survive publication +merely because its map key matches, and an unwind cannot leave a dead actor falsely +advertised as ready. + +### Sub-Decision 8: Replay evidence fails closed + +Vector reconciliation treats strict journal recovery as evidence, not a best-effort +read. A malformed persistence envelope is propagated instead of being classified as +an empty or phantom entity, and an injected truncated-read fault returns an error +instead of a successful prefix. Any load, replay, replacement, or watermark failure +keeps the completion claim absent so restart retries the complete bounded type. + +## Rollout Plan + +1. Pause vector-declaring writes and background vector backfill before the fleet + cutover. Mixed old/new writers are unsafe because an old binary can still perform a + sequence-less replacement that bypasses the new fence. +2. Add the Postgres version, reconciliation-generation, and declaration-authority + tables, including spec-mutation triggers and deletion tombstones. Seed legacy + candidate sequences into generation zero and authority tombstones for vector state + whose spec is already absent. Apply tenant RLS to all new Postgres metadata. Add the + equivalent idempotent Turso bootstrap DDL and deterministic simulation maps. +3. Add tombstone-inclusive journal-stream enumeration for vector repair without + changing active entity-listing semantics. +4. Deploy the generation-and-sequence-carrying trait and backend implementations to + every single and composite writer before resuming background work. The + protocol-revised watermark signature forces one complete ordered rebuild for every + currently or previously declared vector type, including deleted streams and empty + current declaration sets. +5. Confirm the revisioned rebuild and watermark persistence, then resume + vector-declaring writes. Keep the new version table on rollback; it is additive + derived state and protects later re-deployment. + +## Readiness Gates + +- A deterministic stale-backfill/live-write schedule preserves the live vector. +- A newer empty purge cannot be followed by stale vector resurrection. +- A pre-rollout stale candidate belonging to a deleted journal stream is purged and + fenced at the deletion sequence during the revisioned rebuild. +- Remove-all declarations purge and fence the type; intervening writes followed by + re-adding the identical declaration signature trigger a fresh rebuild. +- An older overlapping declaration-set reconciliation cannot obtain a generation, + mutate rows, or publish a watermark after a newer durable declaration completes. +- A delete interrupted after generation allocation resumes after restart, and an + identical later re-add obtains a newer authority revision in both durable stores. +- Beginning a new generation atomically withdraws the previous completion claim, and + an interrupted empty-set reconciliation remains discoverable without that watermark. +- Composite vector updates and deletes advance journal, rows, and the generation-plus- + sequence fence atomically. +- A stale non-vector writer cannot advance the journal after a newer vector declaration + becomes authoritative, in either a single append or an otherwise-valid batch. +- Full replacement persists omission tombstones before registry publication; restart + cannot restore a removed type, and compatibility authority without a catalog row is + still tombstoned. +- Concurrent Postgres and Turso replicas replacing an empty tenant with disjoint + catalogs leave exactly one complete catalog after reopen, never their union. +- An actor that panics or is cancelled after `pre_start` immediately loses readiness; + publication never preserves its dead incarnation. +- Malformed or truncated journal recovery cannot publish a vector completion + watermark from a successful prefix. +- A fresh compatibility store establishes exactly one first-writer declaration and + thereafter obeys the same durable fence as catalog-backed stores. +- Deployment automation prevents sequence-less old writers/backfills from overlapping + sequence-fenced writers during the cutover. +- Equal-sequence replay is idempotent. +- Postgres and Turso integration tests exercise the same ordering contract as the + simulation store. +- A Turso indexed append is atomic under an injected journal/index transaction failure. +- Backfill cannot report or persist completion after a load, replacement, or watermark + failure. +- The full workspace, strict Clippy, determinism, and live local server flows pass. + +## Consequences + +### Positive + +- Vector candidates become monotonic with the journal across live writes, backfill, + replay, deletion, and cleared vectors. +- Turso no longer acknowledges an event while silently dropping its vector update. +- A watermark means the sequence-aware reconciliation actually converged. + +### Negative + +- Indexing backends store one additional small row per reconciled entity. +- Turso spec-derived appends hold an immediate transaction through declaration + validation and, when applicable, vector replacement instead of completing the index + asynchronously. +- An incomplete or revised backfill re-reads the bounded entity type instead of + resuming from row presence. + +### Risks + +- The longer Turso transaction may increase contention. Existing bounded write gates, + append timeouts, and transaction retries remain in force; non-vector appends keep the + optimized single-event path. +- A future backend could incorrectly inherit no-op vector methods. Such a backend must + continue returning no vector watermark authority until it implements this contract. + +### DST Compliance + +- The simulation fence is a `BTreeMap` updated under the same existing store lock as + journal and candidate state. +- Race coverage is an explicit deterministic event order: observe N, commit N+1, + attempt N. It needs no thread, wall clock, or random scheduling. +- The change introduces no ambient I/O, nondeterministic collection, or unbounded + mailbox behavior in simulation-visible crates. + +## Non-Goals + +- Unifying the parallel store implementations; ARN-201 owns that broader contract. +- Producing embedding values, approximate-nearest-neighbor indexes, or ranking changes. +- Making non-indexing EventStore backends authoritative for vector queries. + +## Alternatives Considered + +1. **Compare only candidate-row sequence numbers** — rejected because a newer empty + purge leaves no row that can reject delayed insertion. +2. **Serialize backfill and live writes in the server** — rejected because process + locks do not survive restart and cannot protect independent writers. +3. **Serialize declaration-set backfills only in memory** — rejected as the sole + mechanism because it cannot reject delayed work from another process or a crashed + predecessor. A small local lock is still used to order declaration snapshotting and + generation allocation; the durable generation is authoritative. +4. **Keep Turso write-behind with retry-only recovery** — rejected because retry + exhaustion and a current watermark can make loss permanent. +5. **Add a Turso dirty-row/outbox workflow** — rejected for the current topology + because journal and vector tables already share one transaction manager. It becomes + mandatory if a future backend cannot co-commit them. + +## Rollback Policy + +The version table is additive and may remain populated. Rolling the binary back to an +implementation that performs sequence-less replacement would reopen ARN-216, so a +binary rollback must first pause vector backfill and vector-declaring writes. After +restoring this implementation, rerun the revisioned backfill before resuming traffic.